Package org.archive.net

Examples of org.archive.net.UURI

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        long start = System.currentTimeMillis();
        ArrayList<UURI> list = new ArrayList<UURI>(1000);
        int count = 0;
        final int MAX_COUNT = 1000;
        for (; count < MAX_COUNT; count++) {
          UURI u = UURIFactory.getInstance("http://www" +
              count + ".archive.org/" + count + "/index.html");
          this.filter.add(u.toString(), new CrawlURI(u));
          if (count > 0 && ((count % 100) == 0)) {
            list.add(u);
          }
        }
        this.logger.info("Added " + count + " in " +
            (System.currentTimeMillis() - start));
       
        start = System.currentTimeMillis();
        for (Iterator<UURI> i = list.iterator(); i.hasNext();) {
            UURI uuri = i.next();
            this.filter.add(uuri.toString(), new CrawlURI(uuri));
        }
        this.logger.info("Added random " + list.size() + " in " +
            (System.currentTimeMillis() - start));
       
        start = System.currentTimeMillis();
        for (Iterator<UURI> i = list.iterator(); i.hasNext();) {
            UURI uuri = i.next();
            this.filter.add(uuri.toString(), new CrawlURI(uuri));
        }
        this.logger.info("Deleted random " + list.size() + " in " +
            (System.currentTimeMillis() - start));
        // Looks like delete doesn't work.
        assertTrue("Count is off: " + this.filter.count(),
View Full Code Here
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
        this.uuri = uuri;
        this.pathFromSeed = "";
    }

    public static CrawlURI fromHopsViaString(String uriHopsViaContext) throws URIException {
        UURI u;
        String args[] = uriHopsViaContext.split("\\s+");
        u = UURIFactory.getInstance(args[0]);
        String pathFromSeed = (args.length > 1)?
            args[1].toString() : "";
        UURI via = (args.length > 2 && args[2].length()>1) ?
            UURIFactory.getInstance(args[2].toString()):
            null;
        LinkContext viaContext = (args.length > 3 && args[2].length()>1) ?
                HTMLLinkContext.get(args[3].toString()): null;
        CrawlURI caUri = new CrawlURI(u, pathFromSeed, via, viaContext);
View Full Code Here
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
        return createCrawlURI(destination.toString(), context, hop);
    }

    public CrawlURI createCrawlURI(String destination, LinkContext context, Hop hop)
        throws URIException {
        UURI u = UURIFactory.getInstance(this.getBaseURI(), destination);
        CrawlURI newCaURI = new CrawlURI(
            u,
                extendHopsPath(getPathFromSeed(),
                    hop.getHopChar()),
                this.getUURI(),
View Full Code Here
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
     * lands in the same queue, with the same overlay values, as the
     * triggering URI.
     * @return UURI to use for policy decisions
     */
    public UURI getPolicyBasisUURI() {
        UURI effectiveuuri = null;
        // always use 'via' of prerequisite URIs, if available, so
        // prerequisites go to same queue as trigger URI
        if (getPathFromSeed().endsWith(Hop.PREREQ.getHopString())) {
            effectiveuuri = getVia();
        }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
        return testUri;
    }
    private CrawlURI createTestUri(String urlStr, String via) throws URIException {
       
        UURI testViaUuri = UURIFactory.getInstance(via);
        CrawlURI testUri = createTestUri(urlStr);
        testUri.setVia(testViaUuri);

        return testUri;
    }
View Full Code Here
32
33
34
35
36
37
38
39
40
41
        assertTrue(dr.evaluate(testUri));
    }
   
    private CrawlURI createTestUri(String urlStr) throws URIException{
        UURI testUuri = UURIFactory.getInstance(urlStr);
        CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC);

        return testUri;
    }
View Full Code Here
32
33
34
35
36
37
38
39
40
41
        assertFalse(dr.evaluate(testUri));
    }
   
    private CrawlURI createTestUri(String urlStr) throws URIException{
        UURI testUuri = UURIFactory.getInstance(urlStr);
        CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC);

        return testUri;
    }
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
        assertEquals("http://example.org/original", curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
    }

    protected CrawlURI makeCrawlURI(String uri) throws URIException,
            IOException {
        UURI uuri = UURIFactory.getInstance(uri);
        CrawlURI curi = new CrawlURI(uuri);
        curi.setSeed(true);
        curi.setRecorder(getRecorder());
        return curi;
    }
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
       
        private static String getKey(String url, boolean prefix)
        throws URIException {

                String key = ArchiveUtils.addImpliedHttpIfNecessary(url);
                UURI uuri = UURIFactory.getInstance(key);
                key = uuri.getScheme() + "://" + uuri.getAuthority() +
                        uuri.getEscapedPathQuery();

                key = SURT.fromURI(key);
               
                int hashPos = key.indexOf('#');
                if(hashPos != -1) {
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
     *
     * @param curi CrawlURI to add discoveries to
     * @param wref Link to examine for internal URIs
     */
    protected void extractLink(CrawlURI curi, CrawlURI wref) {
        UURI source = null;
        try {
            source = UURIFactory.getInstance(wref.getURI());
        } catch (URIException e) {
            LOGGER.log(Level.FINE,"bad URI",e);
        }
        if(source == null) {
            // shouldn't happen
            return;
        }
        List<String> found = extractQueryStringLinks(source);
        for (String uri : found) {
            try {
                UURI dest = UURIFactory.getInstance(uri);
                LinkContext lc = LinkContext.SPECULATIVE_MISC;
                Hop hop = Hop.SPECULATIVE;
                addOutlink(curi, dest, lc, hop);
                numberOfLinksExtracted.incrementAndGet();
            } catch (URIException e) {
View Full Code Here
TOP

Related Classes of org.archive.net.UURI

  • org.archive.crawler.datamodel.CrawlURITest
  • org.archive.crawler.frontier.AMQPUrlReceiver$UrlConsumer
  • org.archive.crawler.frontier.URIAuthorityBasedQueueAssignmentPolicy
  • org.archive.crawler.frontier.WagCostAssignmentPolicy
  • org.archive.crawler.io.UriErrorFormatter
  • org.archive.crawler.prefetch.PreconditionEnforcer
  • org.archive.crawler.selftest.SelfTestBase
  • org.archive.crawler.util.BdbUriUniqFilterTest
  • org.archive.crawler.util.BloomUriUniqFilterTest
  • org.archive.crawler.util.FPUriUniqFilterTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.