Package net.fortytwo.twitlogic.persistence

Examples of net.fortytwo.twitlogic.persistence.TweetStore


        try {
            Sail streamingSail = new MockUdpTransactionSail(workingSail);

            try {
                TweetStore store = new TweetStore(streamingSail);
                store.doNotRefreshCoreMetadata();
                store.initialize();

                try {
                    // A connection with which to repeatedly clear the working store
                    final SailConnection c = workingSail.getConnection();

                    try {
                        c.begin();
                        // Offline persister
                        final TweetPersister persister = new TweetPersister(store, null);

                        try {
                            CustomTwitterClient client = new CustomTwitterClient();

                            TweetPersistedLogger pLogger = new TweetPersistedLogger(client.getStatistics(), persister);

                            // Add a "topic sniffer".
                            TopicSniffer topicSniffer = new TopicSniffer(pLogger);

                            // Add a tweet annotator.
                            Matcher matcher = new MultiMatcher(
                                    new DemoAfterthoughtMatcher());

                            final Handler<Tweet> annotator
                                    = new TweetAnnotator(matcher, topicSniffer);

                            Handler<Tweet> adder = new Handler<Tweet>() {
                                public boolean isOpen() {
                                    return annotator.isOpen();
                                }

                                public void handle(final Tweet tweet) throws HandlerException {
                                    try {
                                        c.clear();
                                        c.commit();
                                        c.begin();
                                    } catch (SailException e) {
                                        throw new HandlerException(e);
                                    }

                                     annotator.handle(tweet);
                                }
                            };
                            Handler<Tweet> deleter = new TweetDeleter(store);

                            TweetReceivedLogger rLogger = new TweetReceivedLogger(client.getStatistics(), adder);

                            client.processSampleStream(rLogger, deleter);
                        } finally {
                            persister.close();
                        }
                    } finally {
                        c.rollback();
                        c.close();
                    }
                } finally {
                    store.shutDown();
                }
            } finally {
                streamingSail.shutDown();
            }
        } finally {
View Full Code Here


            Properties props = new Properties();
            props.load(new FileInputStream("/Users/josh/projects/fortytwo/twitlogic/config/twitlogic.properties"));
            TwitLogic.setConfiguration(props);

            // Create a persistent store.
            TweetStore store = new TweetStore();
            store.initialize();

            try {
                TwitterClient client = new Twitter4jClient();

                // Launch linked data server.
                store.startServer(client);

                Object mutex = "";
                synchronized (mutex) {
                    mutex.wait();
                }
            } finally {
                store.shutDown();
            }
        } catch (Throwable t) {
            t.printStackTrace();
        }
    }
View Full Code Here

    }

    private static void mine(final Collection<String> ids) throws Exception {
        CustomTwitterClient client = new CustomTwitterClient();

        TweetStore store = new TweetStore();
        store.initialize();

        try {
            TweetStoreConnection c = store.createConnection();

            try {
                PersistenceContext pc
                        = new PersistenceContext(c);

                PlacePersistenceHelper ph = new PlacePersistenceHelper(pc, client, false);
                for (String id : ids) {
                    Place p = new Place(id);
                    Feature f = pc.persist(p);
                    ph.submit(p, f);
                    c.commit();
                }
            } finally {
                c.rollback();
                c.close();
            }
        } finally {
            store.shutDown();
        }
    }
View Full Code Here

        try {
            Sail streamingSail = new UdpTransactionSail(workingSail, address, ports);

            try {
                TweetStore store = new TweetStore(streamingSail);
                store.doNotRefreshCoreMetadata();
                store.initialize();

                try {
                    // A connection with which to repeatedly clear the working store
                    final SailConnection c = workingSail.getConnection();

                    try {
                        c.begin();

                        // Offline persister
                        final TweetPersister persister = new TweetPersister(store, null);

                        try {
                            CustomTwitterClient client = new CustomTwitterClient();

                            // Note: this is only for serving local files.
                            store.startServer(client);

                            TweetPersistedLogger pLogger = new TweetPersistedLogger(client.getStatistics(), persister);
                            TweetFilterCriterion crit = new TweetFilterCriterion(TwitLogic.getConfiguration());
                            Filter<Tweet> f = new Filter<Tweet>(crit, pLogger);

                            // Add a "topic sniffer".
                            TopicSniffer topicSniffer = new TopicSniffer(f);

                            // Add a tweet annotator.
                            Matcher matcher = new MultiMatcher(
                                    new DemoAfterthoughtMatcher());

                            final Handler<Tweet> annotator
                                    = new TweetAnnotator(matcher, topicSniffer);

                            Handler<Tweet> adder = new Handler<Tweet>() {
                                public boolean isOpen() {
                                    return annotator.isOpen();
                                }

                                public void handle(final Tweet tweet) throws HandlerException {
                                    try {
                                        c.clear();
                                        c.commit();
                                        c.begin();
                                    } catch (SailException e) {
                                        throw new HandlerException(e);
                                    }

                                     annotator.handle(tweet);
                                }
                            };
                            Handler<Tweet> deleter = new TweetDeleter(store);

                            TweetReceivedLogger rLogger = new TweetReceivedLogger(client.getStatistics(), adder);

                            Set<User> users = TwitLogic.findFollowList(client);
                            Set<String> terms = TwitLogic.findTrackTerms();

                            if (0 < users.size() || 0 < terms.size()) {
                                client.processFilterStream(users, terms, null, rLogger, deleter, 0);
                            } else {
                                client.processSampleStream(rLogger, deleter);
                            }
                        } finally {
                            persister.close();
                        }
                    } finally {
                        c.rollback();
                        c.close();
                    }
                } finally {
                    store.shutDown();
                }
            } finally {
                streamingSail.shutDown();
            }
        } finally {
View Full Code Here

    private void testMemoryPersister() throws Exception {
        Sail sail = new MemoryStore();
        sail.initialize();

        try {
            TweetStore store = new TweetStore(sail);
            store.initialize();
            try {
                TweetPersister p = new TweetPersister(store, null);

                stressTest(p, 1000);
            } finally {
                store.shutDown();
            }
        } finally {
            sail.shutDown();
        }
    }
View Full Code Here

    private void testTransientMemoryPersister() throws Exception {
        Sail sail = new MemoryStore();
        sail.initialize();

        try {
            TweetStore store = new TweetStore(sail);
            store.initialize();
            try {
                final SailConnection sc = sail.getConnection();
                try {
                    final TweetPersister p = new TweetPersister(store, null);

                    Handler<Tweet> h = new Handler<Tweet>() {
                        public boolean isOpen() {
                            return p.isOpen();
                        }

                        public void handle(final Tweet tweet) throws HandlerException {
                            try {
                                sc.clear();
                                sc.commit();
                                sc.begin();
                            } catch (SailException e) {
                                throw new HandlerException(e);
                            }
                            p.handle(tweet);
                        }
                    };

                    stressTest(h, 1000);
                } finally {
                    sc.close();
                }
            } finally {
                store.shutDown();
            }
        } finally {
            sail.shutDown();
        }
    }
View Full Code Here

        RecorderSail sail = new RecorderSail(baseSail, out);
        sail.getConfiguration().logReadOperations = false;
        //sail.getConfiguration().logTransactions = false;

        try {
            TweetStore store = new TweetStore(sail);
            store.initialize();
            try {
                final SailConnection sc = baseSail.getConnection();
                try {
                    final TweetPersister p = new TweetPersister(store, null);

                    Handler<Tweet> h = new Handler<Tweet>() {
                        public boolean isOpen() {
                            return p.isOpen();
                        }

                        public void handle(final Tweet tweet) throws HandlerException {
                            try {
                                sc.clear();
                                sc.commit();
                                sc.begin();
                            } catch (SailException e) {
                                throw new HandlerException(e);
                            }
                            p.handle(tweet);
                        }
                    };

                    stressTest(h, 1000);
                } finally {
                    sc.close();
                }
            } finally {
                store.shutDown();
            }
        } finally {
            sail.shutDown();
        }
    }
View Full Code Here

                try {
                    Sail sail = new AGTransactionSail(tSail, rc, commitsPerUpload);
                    try {

                        TweetStore store = new TweetStore(sail);
                        store.doNotRefreshCoreMetadata();
                        store.initialize();
                        try {
                            final SailConnection tc = tSail.getConnection();
                            try {
                                final TweetPersister p = new TweetPersister(store, null);

                                Handler<Tweet> h = new Handler<Tweet>() {
                                    public boolean isOpen() {
                                        return p.isOpen();
                                    }

                                    public void handle(final Tweet tweet) throws HandlerException {
                                        try {
                                            tc.clear();
                                            tc.commit();
                                            tc.begin();
                                        } catch (SailException e) {
                                            throw new HandlerException(e);
                                        }

                                        p.handle(tweet);
                                    }
                                };

                                stressTest(h, 100);
                            } finally {
                                tc.close();
                            }
                        } finally {
                            store.shutDown();
                        }
                    } finally {
                        sail.shutDown();
                    }
                } finally {
View Full Code Here

        transientSail.initialize();

        try {
            Sail sail = new TrivialTransactionSail(transientSail);
            try {
                TweetStore store = new TweetStore(sail);
                store.initialize();

                try {
                    final SailConnection tc = transientSail.getConnection();

                    try {
                        tc.begin();
                        final TweetPersister p = new TweetPersister(store, null);

                        Handler<Tweet> h = new Handler<Tweet>() {
                            public boolean isOpen() {
                                return p.isOpen();
                            }

                            public void handle(final Tweet tweet) throws HandlerException {
                                try {
                                    tc.clear();
                                    tc.commit();
                                    tc.begin();
                                } catch (SailException e) {
                                    throw new HandlerException(e);
                                }
                                p.handle(tweet);
                            }
                        };

                        stressTest(h, 1000);
                    } finally {
                        tc.rollback();
                        tc.close();
                    }
                } finally {
                    store.shutDown();
                }
            } finally {
                sail.shutDown();
            }
        } finally {
View Full Code Here

        Sail sail = new NativeStore(dir);
        sail.initialize();

        try {
            TweetStore store = new TweetStore(sail);
            store.initialize();
            try {
                TweetPersister p = new TweetPersister(store, null);

                stressTest(p, 1000);
            } finally {
                store.shutDown();
            }
        } finally {
            sail.shutDown();
        }
    }
View Full Code Here

TOP

Related Classes of net.fortytwo.twitlogic.persistence.TweetStore

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.