Examples of LaxURI

Examples of org.archive.net.LaxURI

25
26
27
28
29
30
31
32
     * @throws RobotsUnavailableException
     */
    public boolean isRobotPermitted(String url, String userAgent)
            throws IOException, RobotsUnavailableException {
        RobotRules rules = getRulesForUrl(url, userAgent);
        return !rules.blocksPathForUA(new LaxURI(url, false).getPath(),
                userAgent);
    }
View Full Code Here

Examples of org.archive.net.LaxURI

42
43
44
45
46
47
48
49
50
51
52
     */
    public abstract RobotRules getRulesForUrl(String url, String userAgent)
            throws IOException, RobotsUnavailableException;

    public static String robotsUrlForUrl(String url) throws URIException {
        LaxURI uri = new LaxURI(url, false);
        uri.setPath("/robots.txt");
        uri.setQuery(null);
        uri.setFragment(null);
        return uri.toString();
    }
View Full Code Here

Examples of org.archive.net.LaxURI

110
111
112
113
114
115
116
117
118
119
120
    throws LiveDocumentNotAvailableException, URIException, IOException {
   
    HttpClient http = getHttpClient();
    OutputStream os = new FileOutputStream(file);
    ExtendedGetMethod method = new ExtendedGetMethod(os);
    LaxURI lURI = new LaxURI(urlString,true);
    method.setURI(lURI);
    try {
      int code = http.executeMethod(method);
      os.close();
      // TODO: Constant 200
View Full Code Here

Examples of org.archive.net.LaxURI

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
   * @return true if the url-timestamp should not be shown to end users
   */
  public boolean isExcluded(String urlString, String timestamp) {
    SearchResult sr = new SearchResult();

    LaxURI url = null;
    String host = null;
    try {
      url = new LaxURI(urlString,true);
      host = url.getHost();
    } catch (URIException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
      return true;
    }
View Full Code Here

Examples of org.archive.url.LaxURI

139
140
141
142
143
144
145
146
147
148
149
    // then, if that worked, save the recorded value into an ARC
        //     and return it's region
    // if we didn't get a response, forge a fake record and return that.
    try {
      Recorder.setHttpRecorder(recorder);
      LaxURI lURI = new LaxURI(url,true);
      getMethod = new ExtendedGetMethod(url,recorder);
      getMethod.setURI(lURI);
      HttpClient client = getHttpClient();
      getMethod.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
      getMethod.setFollowRedirects(false);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.