Examples of ArcResource


Examples of org.archive.wayback.resourcestore.ArcResource

      LOGGER.info("Cached URL(" + url.toString() + ") in " +
          "ARC(" + name + ") at (" + offset + ")");
      resource = arcCacheDir.getResource(name, offset);
      // add the result to the index:
      if(resource instanceof ArcResource) {
        ArcResource aResource = (ArcResource) resource;
        ARCRecord record = (ARCRecord) aResource.getArcRecord();
       
        SearchResult result = adapter.adapt(record);
        index.addSearchResult(result);
        LOGGER.info("Added URL(" + url.toString() + ") in " +
            "ARC(" + name + ") at (" + offset + ") to LiveIndex");
View Full Code Here

Examples of org.archive.wayback.resourcestore.resourcefile.ArcResource

  protected Resource loadResource(String path, InputStream is) throws IOException, ResourceNotAvailableException
  {
    ArchiveReader archiveReader = ArchiveReaderFactory.get(path, is, false);
   
    if (archiveReader instanceof ARCReader) {
      return new ArcResource((ARCRecord)archiveReader.get(), archiveReader);
    } else if (archiveReader instanceof WARCReader) {
      return new WarcResource((WARCRecord)archiveReader.get(), archiveReader)
    } else {
      throw new IOException("Unknown ArchiveReader");
    }
View Full Code Here

Examples of org.archive.wayback.resourcestore.resourcefile.ArcResource

          ByteArrayInputStream bais = new ByteArrayInputStream(method.getResponseBody());
          ARCRecord r = new ARCRecord(
              new GZIPInputStream(bais),
              "id",0L,false,false,true);
          ArcResource ar = (ArcResource)
            ResourceFactory.ARCArchiveRecordToResource(r, null);
          if(ar.getStatusCode() == 502) {
            throw new LiveDocumentNotAvailableException(urlString);
          } else if(ar.getStatusCode() == 504) {
            throw new LiveWebTimeoutException("Timeout:" + urlString);
          }
          success = true;
          return ar;
         
View Full Code Here

Examples of org.archive.wayback.resourcestore.resourcefile.ArcResource

   
    String urlString = translateRequestPathQuery(httpRequest);
    urlString = UrlOperations.fixupHTTPUrlWithOneSlash(urlString);
    boolean handled = true;
   
    ArcResource r = null;
   
    WaybackRequest wbRequest = new WaybackRequest();
    wbRequest.setAccessPoint(inner);
    wbRequest.setRequestUrl(urlString);

    try {
      String ref = httpRequest.getHeader("Referer");
     
      PerfStats.clearAll();     
     
      if ((ref == null) || !skipHost.matcher(ref).find()) {
        wbRequest.setTimestampSearchKey(true);
        wbRequest.setReplayDate(new Date());
        wbRequest.setReplayRequest();
       
        try {
          inner.queryIndex(wbRequest);
          // Succeeded, so send redirect to query
          httpResponse.sendRedirect(inner.getReplayPrefix() + urlString);
          return true;
        } catch (ResourceIndexNotAvailableException e) {
          throw new LiveDocumentNotAvailableException(e.toString());
        } catch (ResourceNotInArchiveException e) {
          //Continue
        } catch (BadQueryException e) {
          throw new LiveDocumentNotAvailableException(e.toString());
        } catch (AccessControlException e) {
          //Continue
          //throw new LiveDocumentNotAvailableException(e.toString());
        } catch (ConfigurationException e) {
          throw new LiveDocumentNotAvailableException(e.toString());
        }
      }
     
      wbRequest.setLiveWebRequest(true);
     
      if (inner.isEnablePerfStatsHeader()) {
        PerfStats.timeStart(AccessPoint.PerfStat.Total);
        httpResponse = new PerfWritingHttpServletResponse(httpRequest, httpResponse, AccessPoint.PerfStat.Total, inner.getPerfStatsHeader());
      }
     
      Thread.currentThread().setName("Thread " +
          Thread.currentThread().getId() + " " + getBeanName() +
          " handling: " + urlString);
     
      CaptureSearchResult result = new FastCaptureSearchResult();
     
      r = this.getLiveWebResource(result, urlString);
     
      if (r != null) {       
        CaptureSearchResults results = new CaptureSearchResults();
        results.addSearchResult(result);
     
        wbRequest.setReplayTimestamp(result.getCaptureTimestamp());
         
        inner.getReplay().getRenderer(wbRequest, result, r).renderResource(httpRequest, httpResponse, wbRequest, result, r,
            inner.getUriConverter(), results)
      } else {
        throw new LiveDocumentNotAvailableException(urlString);
      }

    } catch(WaybackException e) {
      inner.logError(httpResponse, LIVEWEB_RUNTIME_ERROR_HEADER, e, wbRequest);
      inner.getException().renderException(httpRequest, httpResponse, wbRequest, e, inner.getUriConverter());
   
    } catch(Exception e) {
      inner.logError(httpResponse, LIVEWEB_RUNTIME_ERROR_HEADER, e, wbRequest);
    } finally {
      if (r != null) {
        r.close();
      }
    }
   
    return handled;
  }
View Full Code Here

Examples of org.archive.wayback.resourcestore.resourcefile.ArcResource

  }
 
  protected ArcResource getLiveWebResource(CaptureSearchResult result, String urlString) throws WaybackException, IOException
  {
    URL url = null;
    ArcResource r = null;

    if (!urlString.startsWith(UrlOperations.HTTP_SCHEME) &&
      !urlString.startsWith(UrlOperations.HTTPS_SCHEME)) {
     
      // Remove accidental calendar page requests
      if (urlString.startsWith("*/")) {
        urlString = urlString.substring(2);
      }
     
      // Assume http
      urlString = UrlOperations.HTTP_SCHEME + urlString;
    }
       
    try {
      url = new URL(urlString);
    } catch(MalformedURLException e) {
      throw new BadQueryException("Bad URL(" + urlString + ")");
    }
   
    if ((skipHost != null) && skipHost.matcher(url.getHost()).find()) {
      return null;
    }
   
    if ((dnsCheckTimeout > 0) && !checkUrlDns(url, dnsCheckTimeout)) {
      return null;
    }

    result.setOriginalUrl(urlString);
   
    String canonUrl = urlString;
   
    if (inner.getSelfRedirectCanonicalizer() != null) {
      try {
        canonUrl = inner.getSelfRedirectCanonicalizer().urlStringToKey(urlString);
      } catch (IOException io) {
        throw new BadQueryException("Bad URL(" + urlString + ")");
      }
    }
   
    result.setUrlKey(canonUrl);
   
    // check admin excludes first, if configured:
    if(adminFactory != null) {
      ExclusionFilter f = adminFactory.get();
      if(f == null) {
        LOGGER.severe("Unable to get administrative exclusion filter!");
        throw new AdministrativeAccessControlException(urlString + "is blocked.");
      }
      int ruling = f.filterObject(result);
      if(ruling == ExclusionFilter.FILTER_EXCLUDE) {
        throw new AdministrativeAccessControlException(urlString + "is blocked.");
      }       
    }
    // check robots next, if configured
    if(robotFactory != null) {
      int ruling = robotFactory.get().filterObject(result);
      if(ruling == ExclusionFilter.FILTER_EXCLUDE) {
        throw new RobotAccessControlException(urlString + "is blocked by robots.txt");
      }
    }
    // no robots check, or robots.txt says GO:
    //long start = System.currentTimeMillis();
   
    try {
      PerfStats.timeStart(PerfStat.LiveWeb);
      r = (ArcResource) cache.getCachedResource(url, maxCacheMS , false);
    } finally {
      PerfStats.timeEnd(PerfStat.LiveWeb);
    }

    ARCRecord ar = (ARCRecord) r.getArcRecord();
    int status = ar.getStatusCode();
    if ((status == 200) || ((status >= 300) && (status < 400))) {
      result.setCaptureTimestamp(ar.getMetaData().getDate());
      result.setMimeType(ar.getMetaData().getMimetype());
      return r;
View Full Code Here

Examples of org.archive.wayback.resourcestore.resourcefile.ArcResource

    return false;
  }
 
  private boolean isLiveWebFound(HttpServletRequest request, WaybackRequest wbRequest)
  {
    ArcResource r = null;
   
    String urlString = wbRequest.getRequestUrl();
   
    try {
      r = getLiveWebResource(new FastCaptureSearchResult(), urlString);
      return (r != null);
     
    } catch (Exception e) {
      return false;
    } finally {
      if (r != null) {
        try {
          r.close();
        } catch (IOException e) {
         
        }
      }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.