Package de.jetwick.data

Examples of de.jetwick.data.UrlEntry

139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
            "blup secondnotspamword " + url2,
            "bli secondsomething" + url3};

        JUser user = new JUser("user1");
        JTweet tw1 = new JTweet(1L, tweetsAsStr[0], user).setCreatedAt(new Date(1L));
        tw1.getUrlEntries().add(new UrlEntry(5, 123, url1).setResolvedTitle("title1"));
        JTweet tw2 = new JTweet(2L, tweetsAsStr[1], user).setCreatedAt(new Date(2L));
        tw2.getUrlEntries().add(new UrlEntry(5, 123, url2).setResolvedTitle("title2"));
        JTweet tw3 = new JTweet(3L, tweetsAsStr[2], user).setCreatedAt(new Date(3L));
        tw3.getUrlEntries().add(new UrlEntry(5, 123, url3).setResolvedTitle("title3"));

        execute(Arrays.asList(tw1, tw2, tw3));

        assertEquals(JTweet.QUAL_MAX, tw1.getQuality());
        assertTrue(tw2.getQuality() > JTweet.QUAL_SPAM);
View Full Code Here
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
            if (doFetch) {
                JResult res = fetcher.fetchAndExtract(url, resolveTimeout, false);

                // set resolved url
                if (tweet.getUrlEntries().size() > 0) {
                    UrlEntry ue = tweet.getUrlEntries().iterator().next();
                    ue.setResolvedUrl(res.getUrl());
                    ue.setResolvedTitle(res.getTitle());
                    ue.setResolvedSnippet(res.getText());
                    ue.setResolvedDomain(Helper.extractDomain(url));
                }

                if (urlTitleCleaner.contains(res.getTitle()))
                    tweet.setQuality(20);
View Full Code Here
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
                    }

                    String url = sb.toString();
                    if (url.length() > minLength) {
                        String title = url;
                        UrlEntry entry = urlMap.get(index);
                        if (entry != null) {
                            if (lastIndex == entry.getLastIndex()) {
                                if (!Helper.isEmpty(entry.getResolvedTitle()))
                                    title = Strings.escapeMarkup(entry.getResolvedTitle()).toString();

                                if(entry.getResolvedUrl() != null)
                                    url = entry.getResolvedUrl();
                            }
                        }

                        tmpSb.append(toLink(url, title));
                        return lastIndex;
View Full Code Here
298
299
300
301
302
303
304
305
306
307
308
309
                q.addFilterQuery(ElasticTweetSearch.FIRST_URL_TITLE, name);
                try {
                    List<JTweet> tweets = getTweetSearch().collectObjects(getTweetSearch().query(q.setSize(1)));
                    if (tweets.size() > 0 && tweets.get(0).getUrlEntries().size() > 0) {
                        // TODO there could be more than 1 url!
                        UrlEntry entry = tweets.get(0).getUrlEntries().iterator().next();
                        getRequestCycle().setRequestTarget(new RedirectRequestTarget(entry.getResolvedUrl()));
                    }
                } catch (Exception ex) {
                    logger.error("Error while executing onDirectUrlClick", ex);
                }
            }
View Full Code Here
310
311
312
313
314
315
316
317
318
319
320
321
322
        }

        if (urlCount == 0)
            return new UrlEntry[0];

        UrlEntry urls[] = new UrlEntry[urlCount];
        for (int i = 0; i < urls.length; i++) {
            urls[i] = new UrlEntry();
        }

        for (int counter = 0; counter < urls.length; counter++) {
            String str = (String) source.get("url_pos_" + (counter + 1) + "_s");
            String strs[] = (str).split(",");
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
                    res = new JResult();
                }
                if (res.getUrl().isEmpty())
                    res.setUrl(url);

                UrlEntry entry = new UrlEntry(index, lastIndex, res.getUrl());
                if (res.getTitle().isEmpty())
                    entry.setResolvedTitle(url);
                else
                    entry.setResolvedTitle(res.getTitle());

                entry.setResolvedDomain(Helper.extractDomain(url));
                urlEntries.add(entry);
            }
        }

        return this;
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
        twSearch.forceEmptyQueueAndRefresh(400);

        assertNotNull(twSearch.findByTwitterId(tw.getTwitterId()));
        // original url
        tw = twSearch.findByUrl("http://hiho.de").get(0);
        UrlEntry ue = tw.getUrlEntries().iterator().next();
        assertEquals("http://hiho.de", ue.getOriginalUrl(tw));
        // resolved url
        assertEquals("http://hiho.de_r", ue.getResolvedUrl());
        // fetched title
        assertEquals("http://hiho.de_r_t", ue.getResolvedTitle());
        assertNotNull(twSearch.findByUrl("http://hiho.de_r").get(0));       
    }
View Full Code Here
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
    @Test
    public void testReadUrlEntries() throws IOException {
        JTweet tw = new JTweet(1L, "text", new JUser("peter"));
        List<UrlEntry> entries = new ArrayList<UrlEntry>();

        UrlEntry urlEntry = new UrlEntry(2, 18, "http://fulltest.de/bla");
        urlEntry.setResolvedDomain("resolved-domain.de");
        urlEntry.setResolvedTitle("ResolvedTitel");
        entries.add(urlEntry);

        tw.setUrlEntries(entries);

        XContentBuilder iDoc = twSearch.createDoc(tw);
        String str = iDoc.prettyPrint().string();
        assertTrue(str.contains("\"url_pos_1_s\":\"2,18\""));
        assertTrue(str.contains("\"dest_url_1_s\":\"http://fulltest.de/bla\""));
        assertTrue(str.contains("\"dest_domain_1_s\":\"resolved-domain.de\""));
        assertTrue(str.contains("\"dest_title_1_s\":\"ResolvedTitel\""));

        Map<String, Object> map = new LinkedHashMap<String, Object>();
        map.put("user", "peter");
        map.put("tw", "text");
        map.put("url_i", 1);
        map.put("retw_i", 0);
        map.put("repl_i", 0);
        map.put("url_pos_1_s", "2,18");
        map.put("dest_url_1_s", "http://fulltest.de/bla");
        map.put("dest_domain_1_s", "resolved-domain.de");
        map.put("dest_title_1_s", "ResolvedTitel");

        JTweet tw2 = twSearch.readDoc("1", 0L, map);
        assertEquals(1, tw2.getUrlEntries().size());
        Iterator<UrlEntry> iter = tw2.getUrlEntries().iterator();
        urlEntry = iter.next();
        assertEquals("http://fulltest.de/bla", urlEntry.getResolvedUrl());
        assertEquals("resolved-domain.de", urlEntry.getResolvedDomain());
        assertEquals("ResolvedTitel", urlEntry.getResolvedTitle());
        assertEquals(2, urlEntry.getIndex());
        assertEquals(18, urlEntry.getLastIndex());
    }
View Full Code Here
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
    @Test
    public void testSameUrlTitleButDifferentUrl() throws IOException {
        JTweet tw1 = new JTweet(1L, "text", new JUser("peter"));
        List<UrlEntry> entries = new ArrayList<UrlEntry>();
        UrlEntry urlEntry = new UrlEntry(2, 18, "http://fulltest.de/url2");
        urlEntry.setResolvedDomain("resolved-domain.de");
        urlEntry.setResolvedTitle("ResolvedTitel");
        entries.add(urlEntry);
        tw1.setUrlEntries(entries);

        JTweet tw2 = new JTweet(1L, "text2", new JUser("peter2"));
        entries = new ArrayList<UrlEntry>();
        urlEntry = new UrlEntry(2, 18, "http://fulltest.de/urlNext");
        urlEntry.setResolvedDomain("resolved-domain.de");
        urlEntry.setResolvedTitle("ResolvedTitel");
        entries.add(urlEntry);
        tw2.setUrlEntries(entries);

        twSearch.testUpdate(Arrays.asList(tw1, tw2));
        assertEquals(1, twSearch.query(new TweetQuery()).hits().totalHits());
View Full Code Here
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
    }
   
    @Test
    public void testFindByUrl() {
        List<UrlEntry> entries = new ArrayList<UrlEntry>();
        UrlEntry urlEntry = new UrlEntry(2, 18, "http://fulltest.de/bla");
        urlEntry.setResolvedDomain("resolved-domain.de");
        urlEntry.setResolvedTitle("ResolvedTitel");       
        urlEntry.setIndex(12);
        urlEntry.setLastIndex(26);
        entries.add(urlEntry);

        JTweet tw = createTweet(2L, "test others http://orig.de", "peter2");       
        tw.setUrlEntries(entries);       
        twSearch.update(Collections.singleton(tw), new Date(0), false);
View Full Code Here
TOP

Related Classes of de.jetwick.data.UrlEntry

  • de.jetwick.es.ElasticTweetSearch
  • de.jetwick.es.ElasticTweetSearchTest
  • de.jetwick.snacktory.JResult
  • de.jetwick.tw.cmd.TermCreateCommandTest
  • de.jetwick.tw.Extractor
  • de.jetwick.tw.ExtractorTest
  • de.jetwick.tw.TweetConsumerTest
  • de.jetwick.tw.UrlExtractor
  • de.jetwick.ui.TweetSearchPage
  • de.jetwick.util.GenericUrlResolver
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.