Package org.jsoup.nodes

Examples of org.jsoup.nodes.Document.select()

106
107
108
109
110
111
112
113
114
115
116
        if (doc == null) {
          reporter.incrCounter(this._counterGroup, "Skipped - Unable to Parse HTML", 1);
          return;
        }

        Elements mf = doc.select("[itemtype~=schema.org]");

        if (mf.size() > 0) {
          for (Element e : mf) {
            if (e.hasAttr("itemtype")) {
              output.collect(new Text(e.attr("itemtype").toLowerCase().trim()), new LongWritable(1));
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
            // fsyprint("Fetching %s...", url);
            String last = "";
            final String out = Constants.DATA_PATH + "playlist/"
                + playListName + ".plist";
            final Document doc = Jsoup.connect(url).get();
            final Elements links = doc.select("a[href]");
            final File playListOut = new File(out);
            final FileOutputStream fos = new FileOutputStream(
                playListOut);
            final BufferedWriter bw = new BufferedWriter(
                new OutputStreamWriter(fos));
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    public JsonObject extractTweet(String html)
  throws java.net.MalformedURLException, java.io.UnsupportedEncodingException {
  JsonObject status = new JsonObject();

  Document doc = Jsoup.parse(html);
  Element tweet_div = doc.select("div.permalink-tweet").first();

  String tweet_text = tweet_div.select("p.tweet-text").first().text();
  status.addProperty("text", tweet_text);

  String tweet_id = tweet_div.attr("data-tweet-id");
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
  String tweet_id = tweet_div.attr("data-tweet-id");
  status.addProperty("id_str", tweet_id);
  status.addProperty("id", Long.parseLong(tweet_id));

  String timestamp = doc.select("span.js-short-timestamp").first().attr("data-time");
  Date created_at = new Date();
  created_at.setTime(Long.parseLong(timestamp) * 1000);
  status.addProperty("created_at", date_fmt.format(created_at));

  Elements js_stats_retweets = doc.select("li.js-stat-retweets");
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
  String timestamp = doc.select("span.js-short-timestamp").first().attr("data-time");
  Date created_at = new Date();
  created_at.setTime(Long.parseLong(timestamp) * 1000);
  status.addProperty("created_at", date_fmt.format(created_at));

  Elements js_stats_retweets = doc.select("li.js-stat-retweets");
  if (!js_stats_retweets.isEmpty()) {
      status.addProperty("retweeted", true);
      String count = js_stats_retweets.select("strong").first().text();
      status.addProperty("retweet_count", Long.parseLong(count));
  } else {
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
      status.addProperty("retweet_count", Long.parseLong(count));
  } else {
      status.addProperty("retweeted", false);
      status.addProperty("retweet_count", 0);
  }
  Elements js_stats_favs = doc.select("li.js-stat-favorites");
  status.addProperty("favorited", !js_stats_favs.isEmpty());
     

  // User subfield
  JsonObject user = new JsonObject();
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
  user.addProperty("name", user_name);
 
  status.add("user", user);
 
  // Geo information
  Elements tweet_loc = doc.select("a.tweet-geo-text");
  if (!tweet_loc.isEmpty()) {
      JsonObject location = new JsonObject();
      Element loc = tweet_loc.first();
      // Adding http to avoid malformed URL exception
      URL url = new URL("http:" + loc.attr("href"));
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
            // print("Fetching %s...", url);
            String last = "";
            final String out = Constants.DATA_PATH + "playlist/"
                + playListName + ".plist";
            final Document doc = Jsoup.connect(url).get();
            final Elements links = doc.select("iframe");

            final File playListOut = new File(out);
            final FileOutputStream fos = new FileOutputStream(
                playListOut);
            final BufferedWriter bw = new BufferedWriter(
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
            // print("Fetching %s...", url);
            String last = "";
            final String out = Constants.DATA_PATH + "playlist/"
                + playListName + ".plist";
            final Document doc = Jsoup.connect(url).get();
            final Elements links = doc.select("a[href]");
            final File playListOut = new File(out);
            final FileOutputStream fos = new FileOutputStream(
                playListOut);
            final BufferedWriter bw = new BufferedWriter(
                new OutputStreamWriter(fos));
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
  }

  public void transformString(String channelString) {
    Document doc = Jsoup.parse(channelString);
    Elements tmp;
    tmp = doc.select("alias");
    if (tmp != null) {
      this.alias = (tmp.text());
    }
    tmp = doc.select("thumbImageUrl");
    if (tmp != null) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.