Package org.archive.modules

Examples of org.archive.modules.CrawlURI

234
235
236
237
238
239
240
241
242
243

    protected CrawlURI makeCrawlURI(String uri) throws URIException,
            IOException {
        UURI uuri = UURIFactory.getInstance(uri);
        CrawlURI curi = new CrawlURI(uuri);
        curi.setSeed(true);
        curi.setRecorder(getRecorder());
        return curi;
    }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
     * @param policy
     * @param r
     * @throws IOException
     */
    public void evalQueryString(RobotsPolicy policy, Robotstxt rtxt) throws IOException {
        CrawlURI qs = new CrawlURI(UURIFactory.getInstance("http://example.com/ok?butno=something"));

        assertFalse("ignoring query-string", policy.allows("Mozilla allowbot2 99.9", qs, rtxt));
       
    }
View Full Code Here
16
17
18
19
20
21
22
23
24
    public void testNoVia() throws Exception {
        ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
        List<String> surtPrefixes = new ArrayList<String>();
        surtPrefixes.add("http://(org,archive,");
        dr.setSurtPrefixes(surtPrefixes);
        CrawlURI testUri = createTestUri("http://example.com");
       
        assertFalse(dr.evaluate(testUri));
    }
View Full Code Here
24
25
26
27
28
29
30
31
32
    }
    public void testNoSurts() throws Exception {
        ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
        List<String> surtPrefixes = new ArrayList<String>();
        dr.setSurtPrefixes(surtPrefixes);
        CrawlURI testUri = createTestUri("http://example.com");

        assertFalse(dr.evaluate(testUri));
    }
View Full Code Here
31
32
33
34
35
36
37
38
39
        assertFalse(dr.evaluate(testUri));
    }
    public void testNullSurts() throws Exception {
        ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
        dr.setSurtPrefixes(null);
        CrawlURI testUri = createTestUri("http://example.com");

        assertFalse(dr.evaluate(testUri));
    }
View Full Code Here
40
41
42
43
44
45
46
47
48
    public void testPositiveSingleSurt() throws Exception {
        ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
        List<String> surtPrefixes = new ArrayList<String>();
        surtPrefixes.add("http://(org,archive,");
        dr.setSurtPrefixes(surtPrefixes);
        CrawlURI testUri = createTestUri("http://example.com","http://archive.org");

        assertTrue(dr.evaluate(testUri));
    }
View Full Code Here
49
50
51
52
53
54
55
56
57
    public void testNegativeSingleSurt() throws Exception {
        ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
        List<String> surtPrefixes = new ArrayList<String>();
        surtPrefixes.add("http://(org,archive,");
        dr.setSurtPrefixes(surtPrefixes);
        CrawlURI testUri = createTestUri("http://example.com","http://google.com");

        assertFalse(dr.evaluate(testUri));
    }
View Full Code Here
60
61
62
63
64
65
66
67
68
        List<String> surtPrefixes = new ArrayList<String>();
        surtPrefixes.add("http://(org,archive,");
        surtPrefixes.add("http://(com,test,");
        surtPrefixes.add("http://(com,google,");
        dr.setSurtPrefixes(surtPrefixes);
        CrawlURI testUri = createTestUri("http://example.com","http://google.com");

        assertTrue(dr.evaluate(testUri));
    }
View Full Code Here
71
72
73
74
75
76
77
78
79
        List<String> surtPrefixes = new ArrayList<String>();
        surtPrefixes.add("http://(org,archive,");
        surtPrefixes.add("http://(com,google,");
        surtPrefixes.add("http://(com,test,");
        dr.setSurtPrefixes(surtPrefixes);
        CrawlURI testUri = createTestUri("http://example.com","http://google.com");

        assertTrue(dr.evaluate(testUri));
    }
View Full Code Here
82
83
84
85
86
87
88
89
90
        List<String> surtPrefixes = new ArrayList<String>();
        surtPrefixes.add("http://(org,archive,");
        surtPrefixes.add("http://(com,test,");
        surtPrefixes.add("http://(com,google,");
        dr.setSurtPrefixes(surtPrefixes);
        CrawlURI testUri = createTestUri("http://example.com","http://negativeexample.com");

        assertFalse(dr.evaluate(testUri));
    }
View Full Code Here
TOP

Related Classes of org.archive.modules.CrawlURI

  • org.archive.crawler.datamodel.CandidateURITest
  • org.archive.crawler.datamodel.CrawlURITest
  • org.archive.crawler.deciderules.ClassKeyMatchesRegexDecideRule
  • org.archive.crawler.framework.ToeThread
  • org.archive.crawler.frontier.AbstractFrontier
  • org.archive.crawler.frontier.AMQPUrlReceiver$UrlConsumer
  • org.archive.crawler.frontier.BdbMultipleWorkQueues
  • org.archive.crawler.frontier.BdbMultipleWorkQueuesTest
  • org.archive.crawler.frontier.BdbWorkQueue
  • org.archive.crawler.frontier.FrontierJournal
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.