Package twitter4j

Examples of twitter4j.TwitterStream$StreamHandlingThread


    cb.setDebugEnabled(true)
      .setOAuthConsumerKey("*********")
      .setOAuthConsumerSecret("************")
      .setOAuthAccessToken("*******************")
      .setOAuthAccessTokenSecret("****************");
    TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
        TwitterDownloader td = new TwitterDownloader();
        StatusListener listener = td.makeListener(twitterStream);
        twitterStream.addListener(listener);
        twitterStream.sample();
  }
View Full Code Here


      }

    };

    TwitterStream twitterStream = new TwitterStreamFactory(
        new ConfigurationBuilder().setJSONStoreEnabled(true).build())
        .getInstance();

    twitterStream.addListener(listener);
    twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    AccessToken token = new AccessToken(accessToken, accessTokenSecret);
    twitterStream.setOAuthAccessToken(token);
   
    if (keyWords.length == 0) {

      twitterStream.sample();
    }

    else {

      FilterQuery query = new FilterQuery().track(keyWords);
      twitterStream.filter(query);
    }

  }
View Full Code Here

      {
        log.info("Cleanup_twitter_stream");
      }
    }; // ConnectionLifeCycleListener

    final TwitterStream twitterStream;
    final StatusListener statusListener;
    final int QUEUE_SIZE = 2000;
    /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread.   */
    final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE);
    final LinkedList<String> dimensions = new LinkedList<String>();
    final long startMsec = System.currentTimeMillis();

    dimensions.add("htags");
    dimensions.add("lang");
    dimensions.add("utc_offset");

    //
    //   set up Twitter Spritzer
    //
    twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener);
    statusListener = new StatusListener() {  // This is what really gets called to deliver stuff from twitter4j
      @Override
      public void onStatus(Status status)
      {
        // time to stop?
        if (Thread.currentThread().isInterrupted()) {
          throw new RuntimeException("Interrupted, time to stop");
        }
        try {
          boolean success = queue.offer(status, 15L, TimeUnit.SECONDS);
          if (!success) {
            log.warn("queue too slow!");
          }
        } catch (InterruptedException e) {
          throw new RuntimeException("InterruptedException", e);
        }
      }

      @Override
      public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice)
      {
        //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
      }

      @Override
      public void onTrackLimitationNotice(int numberOfLimitedStatuses)
      {
        // This notice will be sent each time a limited stream becomes unlimited.
        // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity.
        log.warn("Got track limitation notice:" + numberOfLimitedStatuses);
      }

      @Override
      public void onScrubGeo(long userId, long upToStatusId)
      {
        //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
      }

      @Override
      public void onException(Exception ex)
      {
        ex.printStackTrace();
      }

      @Override
      public void onStallWarning(StallWarning warning) {
        System.out.println("Got stall warning:" + warning);
      }
    };

    twitterStream.addListener(statusListener);
    twitterStream.sample(); // creates a generic StatusStream
    log.info("returned from sample()");

    return new Firehose() {

      private final Runnable doNothingRunnable = new Runnable() {
        public void run()
        {
        }
      };

      private long rowCount = 0L;
      private boolean waitIfmax = (maxEventCount < 0L);
      private final Map<String, Object> theMap = new HashMap<String, Object>(2);
      // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper();

      private boolean maxTimeReached()
      {
        if (maxRunMinutes <= 0) {
          return false;
        } else {
          return (System.currentTimeMillis() - startMsec) / 60000L >= maxRunMinutes;
        }
      }

      private boolean maxCountReached()
      {
        return maxEventCount >= 0 && rowCount >= maxEventCount;
      }

      @Override
      public boolean hasMore()
      {
        if (maxCountReached() || maxTimeReached()) {
          return waitIfmax;
        } else {
          return true;
        }
      }

      @Override
      public InputRow nextRow()
      {
        // Interrupted to stop?
        if (Thread.currentThread().isInterrupted()) {
          throw new RuntimeException("Interrupted, time to stop");
        }

        // all done?
        if (maxCountReached() || maxTimeReached()) {
          if (waitIfmax) {
            // sleep a long time instead of terminating
            try {
              log.info("reached limit, sleeping a long time...");
              sleep(2000000000L);
            } catch (InterruptedException e) {
              throw new RuntimeException("InterruptedException", e);
            }
          } else {
            // allow this event through, and the next hasMore() call will be false
          }
        }
        if (++rowCount % 1000 == 0) {
          log.info("nextRow() has returned %,d InputRows", rowCount);
        }

        Status status;
        try {
          status = queue.take();
        } catch (InterruptedException e) {
          throw new RuntimeException("InterruptedException", e);
        }

        HashtagEntity[] hts = status.getHashtagEntities();
        if (hts != null && hts.length > 0) {
          List<String> hashTags = Lists.newArrayListWithExpectedSize(hts.length);
          for (HashtagEntity ht : hts) {
            hashTags.add(ht.getText());
          }

          theMap.put("htags", Arrays.asList(hashTags.get(0)));
        }

        long retweetCount = status.getRetweetCount();
        theMap.put("retweet_count", retweetCount);
        User user = status.getUser();
        if (user != null) {
          theMap.put("follower_count", user.getFollowersCount());
          theMap.put("friends_count", user.getFriendsCount());
          theMap.put("lang", user.getLang());
          theMap.put("utc_offset", user.getUtcOffset())// resolution in seconds, -1 if not available?
          theMap.put("statuses_count", user.getStatusesCount());
        }

        return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap);
      }

      @Override
      public Runnable commit()
      {
        // ephemera in, ephemera out.
        return doNothingRunnable; // reuse the same object each time
      }

      @Override
      public void close() throws IOException
      {
        log.info("CLOSE twitterstream");
        twitterStream.shutdown(); // invokes twitterStream.cleanUp()
      }
    };
  }
View Full Code Here

    listener = new TwitterStreamListener(redisDB);

    /** Number of old tweets to catch before starting to listen live stream */
    int count = 0;

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addListener(listener);
    FilterQuery filterQuery = new FilterQuery();
    filterQuery.count(count);
    if (args.length == 0) {
      args = new String[1];
      args[0] = "#twitter";
    }
    filterQuery.track(args);
    twitterStream.filter(filterQuery);
  }
View Full Code Here

        return maxId;
    }

    public TwitterStream streamingTwitter(Collection<String> track, final Queue<JTweet> queue) throws TwitterException {
        String[] trackArray = track.toArray(new String[track.size()]);
        TwitterStream stream = new TwitterStreamFactory().getInstance(twitter.getAuthorization());
        stream.addListener(new StatusListener() {

            @Override
            public void onStatus(Status status) {
                // ugly twitter ...
                if (Helper.isEmpty(status.getUser().getScreenName()))
                    return;

                if (!queue.offer(new JTweet(toTweet(status), new JUser(status.getUser()))))
                    logger.error("Cannot add tweet as input queue for streaming is full:" + queue.size());
            }

            @Override
            public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
                logger.error("We do not support onDeletionNotice at the moment! Tweet id: "
                        + statusDeletionNotice.getStatusId());
            }

            @Override
            public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
                logger.warn("onTrackLimitationNotice:" + numberOfLimitedStatuses);
            }

            @Override
            public void onException(Exception ex) {
                logger.error("onException", ex);
            }

            @Override
            public void onScrubGeo(long userId, long upToStatusId) {
            }
        });
        stream.filter(new FilterQuery(0, new long[0], trackArray));
        return stream;
    }
View Full Code Here

        this.resultTweets = packages;
    }

    @Override
    public void run() {
        TwitterStream stream = null;
        TwitterStream oldStream = null;
        // we cannot detect frequency of all terms but detect + remove high frequent disturbers
        Map<String, Integer> termFreq = new LinkedHashMap<String, Integer>();

        while (true) {
            try {
                // stream only LESS FREQUENT tags! leave popular tags only for search               
                Collection<String> input = initTags(termFreq);
                termFreq.clear();
                if (input.isEmpty()) {
                    logger.error("No less frequent tags found! Frequency limit:" + tweetsPerSecLimit);
                    if (!myWait(10))
                        break;
                    continue;
                }
                int counter = 0;
                logger.info("Starting over with " + input.size()
                        + " tags. indexed tweets:" + counter
                        + " tweetsPerSecLimit:" + tweetsPerSecLimit
                        + " " + input);
                if (stream != null)
                    oldStream = stream;

                // use a separate collection here to let the listener release when doing garbage collection
                // (the listener which is added in the streamingTwitter method)
                BlockingQueue<JTweet> queue = new LinkedBlockingQueue<JTweet>(1000);
                stream = twSearch.streamingTwitter(input, queue);

                // shutdown old stream
                if (oldStream != null) {
                    oldStream.shutdown();
//                    oldStream.cleanUp();
                }

                long start = System.currentTimeMillis();
                while (true) {
View Full Code Here

            query.locations(loc);
            */
        }

        TwitterStream stream = streamFactory.getInstance();
        stream.addListener(new InnerStatusHandler(stream, addHandler, deleteHandler));
        stream.filter(query);

        waitIndefinitely();
    }
View Full Code Here

        waitIndefinitely();
    }

    // note: streaming methods are not synchronized
    public void processSampleStream(Handler<Tweet> addHandler, Handler<Tweet> deleteHandler) throws TwitterClientException {
        TwitterStream stream = streamFactory.getInstance();
        stream.addListener(new InnerStatusHandler(stream, addHandler, deleteHandler));
        stream.sample();

        waitIndefinitely();
    }
View Full Code Here

        }
        twitterProperties.load(new FileInputStream(twitter4jPropsFile));

        cb.setDebugEnabled(Boolean.valueOf(twitterProperties.getProperty("debug")))
                .setUser(twitterProperties.getProperty("user")).setPassword(twitterProperties.getProperty("password"));
        TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
        StatusListener statusListener = new StatusListener() {

            @Override
            public void onException(Exception ex) {
                logger.error("error", ex);
            }

            @Override
            public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
                logger.error("error");
            }

            @Override
            public void onStatus(Status status) {
                messageQueue.add(status);

            }

            @Override
            public void onScrubGeo(long userId, long upToStatusId) {
                logger.error("error");
            }

            @Override
            public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
                logger.error("error");
            }
        };
        twitterStream.addListener(statusListener);
        twitterStream.sample();

    }
View Full Code Here

                public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
                public void onException(Exception ex) {
                    ex.printStackTrace();
                }
         };
         TwitterStream twitterStream = new TwitterStreamFactory(listener).getInstance("user","password");
         twitterStream.sample();
        
         Thread.sleep(50000);
      
    }
View Full Code Here

TOP

Related Classes of twitter4j.TwitterStream$StreamHandlingThread

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.