Examples of RobotAccessControlException

Examples of org.archive.wayback.exception.RobotAccessControlException

89
90
91
92
93
94
95
96
97
98
99
    if(getLiveWebGone()) {
      throw new RobotNotAvailableException("The URL " + requestUrl +
      " is blocked by the sites robots.txt file");
    }
    if(isSawRobots() && !isPassedRobots()) {
      throw new RobotAccessControlException("The URL " + requestUrl +
          " is blocked by the sites robots.txt file");
    }
    if(isSawAdministrative() && !isPassedAdministrative()) {
      throw new AdministrativeAccessControlException(requestUrl +
          "  is not available in the Wayback Machine.");
View Full Code Here

Examples of org.archive.wayback.exception.RobotAccessControlException

228
229
230
231
232
233
234
235
236
237
238
    }
    // check robots next, if configured
    if(robotFactory != null) {
      int ruling = robotFactory.get().filterObject(result);
      if(ruling == ExclusionFilter.FILTER_EXCLUDE) {
        throw new RobotAccessControlException(urlString + "is blocked by robots.txt");
      }
    }
    // no robots check, or robots.txt says GO:
    //long start = System.currentTimeMillis();
   
View Full Code Here

Examples of org.archive.wayback.exception.RobotAccessControlException

85
86
87
88
89
90
91
92
93
94
95
      status = robotsFilter.filterObject(resultTester);
    }
   
    if (status != ExclusionFilter.FILTER_INCLUDE) {
      if (throwOnFail) {
        throw new RuntimeIOException(403, new RobotAccessControlException(resultTester.getOriginalUrl() + " is blocked by the sites robots.txt file"));
      } else {
        lastKey = resultTester.getUrlKey();
        return cachedValue;
      }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.