/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package org.apache.shindig.gadgets.http;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.HttpVersion;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.client.protocol.RequestAddCookies;
import org.apache.http.client.protocol.ResponseProcessCookies;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.ConnectionPoolTimeoutException;
import org.apache.http.conn.HttpHostConnectException;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.conn.ProxySelectorRoutePlanner;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.ByteArrayBuffer;
import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.gadgets.GadgetException;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.net.ProxySelector;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import java.util.zip.Inflater;
import java.util.zip.InflaterInputStream;
import javax.annotation.Nullable;
import javax.servlet.http.HttpServletResponse;
/**
* A simple HTTP fetcher implementation based on Apache httpclient. Not recommended for production deployments until
* the following issues are addressed:
* <p/>
* 1. This class potentially allows access to resources behind an organization's firewall.
* 2. This class does not enforce any limits on what is fetched from remote hosts.
*/
@Singleton
public class BasicHttpFetcher implements HttpFetcher {
private static final int DEFAULT_CONNECT_TIMEOUT_MS = 5000;
private static final int DEFAULT_READ_TIMEOUT_MS = 5000;
private static final int DEFAULT_MAX_OBJECT_SIZE = 0; // no limit
private static final long DEFAULT_SLOW_RESPONSE_WARNING = 10000;
protected final HttpClient FETCHER;
// mutable fields must be volatile
private volatile int maxObjSize;
private volatile long slowResponseWarning;
private static final Logger LOG = Logger.getLogger(BasicHttpFetcher.class.getName());
private final Set<Class<?>> TIMEOUT_EXCEPTIONS = ImmutableSet.<Class<?>>of(ConnectionPoolTimeoutException.class,
SocketTimeoutException.class, SocketException.class, HttpHostConnectException.class, NoHttpResponseException.class,
InterruptedException.class, UnknownHostException.class);
/**
* Creates a new fetcher using the default maximum object size and timeout --
* no limit and 5 seconds.
* @param basicHttpFetcherProxy The http proxy to use.
*/
@Inject
public BasicHttpFetcher(@Nullable @Named("org.apache.shindig.gadgets.http.basicHttpFetcherProxy")
String basicHttpFetcherProxy) {
this(DEFAULT_MAX_OBJECT_SIZE, DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_READ_TIMEOUT_MS,
basicHttpFetcherProxy);
}
/**
* Creates a new fetcher for fetching HTTP objects. Not really suitable
* for production use. Use of an HTTP proxy for security is also necessary
* for production deployment.
*
* @param maxObjSize Maximum size, in bytes, of the object we will fetch, 0 if no limit..
* @param connectionTimeoutMs timeout, in milliseconds, for connecting to hosts.
* @param readTimeoutMs timeout, in millseconds, for unresponsive connections
* @param basicHttpFetcherProxy The http proxy to use.
*/
public BasicHttpFetcher(int maxObjSize, int connectionTimeoutMs, int readTimeoutMs,
String basicHttpFetcherProxy) {
// Create and initialize HTTP parameters
setMaxObjectSizeBytes(maxObjSize);
setSlowResponseWarning(DEFAULT_SLOW_RESPONSE_WARNING);
HttpParams params = new BasicHttpParams();
ConnManagerParams.setTimeout(params, connectionTimeoutMs);
// These are probably overkill for most sites.
ConnManagerParams.setMaxTotalConnections(params, 1152);
ConnManagerParams.setMaxConnectionsPerRoute(params, new ConnPerRouteBean(256));
HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
HttpProtocolParams.setUserAgent(params, "Apache Shindig");
HttpProtocolParams.setContentCharset(params, "UTF-8");
HttpConnectionParams.setConnectionTimeout(params, connectionTimeoutMs);
HttpConnectionParams.setSoTimeout(params, readTimeoutMs);
HttpConnectionParams.setStaleCheckingEnabled(params, true);
HttpClientParams.setRedirecting(params, true);
HttpClientParams.setAuthenticating(params, false);
// Create and initialize scheme registry
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443));
ClientConnectionManager cm = new ThreadSafeClientConnManager(params, schemeRegistry);
DefaultHttpClient client = new DefaultHttpClient(cm, params);
// Set proxy if set via guice.
if (!StringUtils.isEmpty(basicHttpFetcherProxy)) {
String[] splits = basicHttpFetcherProxy.split(":");
ConnRouteParams.setDefaultProxy(
client.getParams(), new HttpHost(splits[0], Integer.parseInt(splits[1]), "http"));
}
// try resending the request once
client.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(1, true));
// Add hooks for gzip/deflate
client.addRequestInterceptor(new HttpRequestInterceptor() {
public void process(
final org.apache.http.HttpRequest request,
final HttpContext context) throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip, deflate");
}
}
});
client.addResponseInterceptor(new HttpResponseInterceptor() {
public void process(
final org.apache.http.HttpResponse response,
final HttpContext context) throws HttpException, IOException {
HttpEntity entity = response.getEntity();
if (entity != null) {
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
for (HeaderElement codec : ceheader.getElements()) {
String codecname = codec.getName();
if ("gzip".equalsIgnoreCase(codecname)) {
response.setEntity(
new GzipDecompressingEntity(response.getEntity()));
return;
} else if ("deflate".equals(codecname)) {
response.setEntity(new DeflateDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
});
client.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler() );
// Disable automatic storage and sending of cookies (see SHINDIG-1382)
client.removeRequestInterceptorByClass(RequestAddCookies.class);
client.removeResponseInterceptorByClass(ResponseProcessCookies.class);
// Use Java's built-in proxy logic in case no proxy set via guice.
if (StringUtils.isEmpty(basicHttpFetcherProxy)) {
ProxySelectorRoutePlanner routePlanner = new ProxySelectorRoutePlanner(
client.getConnectionManager().getSchemeRegistry(),
ProxySelector.getDefault());
client.setRoutePlanner(routePlanner);
}
FETCHER = client;
}
static class GzipDecompressingEntity extends HttpEntityWrapper {
public GzipDecompressingEntity(final HttpEntity entity) {
super(entity);
}
@Override
public InputStream getContent() throws IOException, IllegalStateException {
// the wrapped entity's getContent() decides about repeatability
InputStream wrappedin = wrappedEntity.getContent();
return new GZIPInputStream(wrappedin);
}
@Override
public long getContentLength() {
// length of ungzipped content is not known
return -1;
}
}
static class DeflateDecompressingEntity extends HttpEntityWrapper {
public DeflateDecompressingEntity(final HttpEntity entity) {
super(entity);
}
@Override
public InputStream getContent()
throws IOException, IllegalStateException {
// the wrapped entity's getContent() decides about repeatability
InputStream wrappedin = wrappedEntity.getContent();
return new InflaterInputStream(wrappedin, new Inflater(true));
}
@Override
public long getContentLength() {
// length of ungzipped content is not known
return -1;
}
}
public HttpResponse fetch(org.apache.shindig.gadgets.http.HttpRequest request)
throws GadgetException {
HttpUriRequest httpMethod = null;
Preconditions.checkNotNull(request);
final String methodType = request.getMethod();
final org.apache.http.HttpResponse response;
final long started = System.currentTimeMillis();
// Break the request Uri to its components:
Uri uri = request.getUri();
if (StringUtils.isEmpty(uri.getAuthority())) {
throw new GadgetException(GadgetException.Code.INVALID_USER_DATA,
"Missing domain name for request: " + uri,
HttpServletResponse.SC_BAD_REQUEST);
}
if (StringUtils.isEmpty(uri.getScheme())) {
throw new GadgetException(GadgetException.Code.INVALID_USER_DATA,
"Missing schema for request: " + uri,
HttpServletResponse.SC_BAD_REQUEST);
}
String[] hostparts = StringUtils.splitPreserveAllTokens(uri.getAuthority(),':');
int port = -1; // default port
if (hostparts.length > 2) {
throw new GadgetException(GadgetException.Code.INVALID_USER_DATA,
"Bad host name in request: " + uri.getAuthority(),
HttpServletResponse.SC_BAD_REQUEST);
}
if (hostparts.length == 2) {
try {
port = Integer.parseInt(hostparts[1]);
} catch (NumberFormatException e) {
throw new GadgetException(GadgetException.Code.INVALID_USER_DATA,
"Bad port number in request: " + uri.getAuthority(),
HttpServletResponse.SC_BAD_REQUEST);
}
}
String requestUri = uri.getPath();
// Treat path as / if set as null.
if (uri.getPath() == null) {
requestUri = "/";
}
if (uri.getQuery() != null) {
requestUri += '?' + uri.getQuery();
}
// Get the http host to connect to.
HttpHost host = new HttpHost(hostparts[0], port, uri.getScheme());
try {
if ("POST".equals(methodType) || "PUT".equals(methodType)) {
HttpEntityEnclosingRequestBase enclosingMethod = ("POST".equals(methodType))
? new HttpPost(requestUri)
: new HttpPut(requestUri);
if (request.getPostBodyLength() > 0) {
enclosingMethod.setEntity(new InputStreamEntity(request.getPostBody(), request.getPostBodyLength()));
}
httpMethod = enclosingMethod;
} else if ("GET".equals(methodType)) {
httpMethod = new HttpGet(requestUri);
} else if ("HEAD".equals(methodType)) {
httpMethod = new HttpHead(requestUri);
} else if ("DELETE".equals(methodType)) {
httpMethod = new HttpDelete(requestUri);
}
for (Map.Entry<String, List<String>> entry : request.getHeaders().entrySet()) {
httpMethod.addHeader(entry.getKey(), StringUtils.join(entry.getValue(), ','));
}
// Disable following redirects.
if (!request.getFollowRedirects()) {
httpMethod.getParams().setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, false);
}
// HttpClient doesn't handle all cases when breaking url (specifically '_' in domain)
// So lets pass it the url parsed:
response = FETCHER.execute(host, httpMethod);
if (response == null) {
throw new IOException("Unknown problem with request");
}
long now = System.currentTimeMillis();
if (now - started > slowResponseWarning) {
slowResponseWarning(request, started, now);
}
return makeResponse(response);
} catch (Exception e) {
long now = System.currentTimeMillis();
// Find timeout exceptions, respond accordingly
if (TIMEOUT_EXCEPTIONS.contains(e.getClass())) {
LOG.info("Timeout for " + request.getUri() + " Exception: " + e.getClass().getName() + " - " + e.getMessage() + " - " + (now - started) + "ms");
return HttpResponse.timeout();
}
LOG.log(Level.INFO, "Got Exception fetching " + request.getUri() + " - " + (now - started) + "ms", e);
// Separate shindig error from external error
throw new GadgetException(GadgetException.Code.INTERNAL_SERVER_ERROR, e,
HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
} finally {
// cleanup any outstanding resources..
if (httpMethod != null) try {
httpMethod.abort();
} catch (UnsupportedOperationException e) {
// ignore
}
}
}
/**
* Called when a request takes too long. Consider subclassing this if you want to do something other than logging
* a warning .
*
* @param request the request that generated the slowrequest
* @param started the time the request started, in milliseconds.
* @param finished the time the request finished, in milliseconds.
*/
protected void slowResponseWarning(HttpRequest request, long started, long finished) {
LOG.warning("Slow response from " + request.getUri() + ' ' + (finished - started) + "ms");
}
/**
* Change the global maximum fetch size (in bytes) for all fetches.
*
* @param maxObjectSizeBytes value for maximum number of bytes, or 0 for no limit
*/
@Inject(optional = true)
public void setMaxObjectSizeBytes(@Named("shindig.http.client.max-object-size-bytes") int maxObjectSizeBytes) {
this.maxObjSize = maxObjectSizeBytes;
}
/**
* Change the global threshold for warning about slow responses
*
* @param slowResponseWarning time in milliseconds after we issue a warning
*/
@Inject(optional = true)
public void setSlowResponseWarning(@Named("shindig.http.client.slow-response-warning") long slowResponseWarning) {
this.slowResponseWarning = slowResponseWarning;
}
/**
* Change the global connection timeout for all new fetchs.
*
* @param connectionTimeoutMs new connection timeout in milliseconds
*/
@Inject(optional = true)
public void setConnectionTimeoutMs(@Named("shindig.http.client.connection-timeout-ms") int connectionTimeoutMs) {
Preconditions.checkArgument(connectionTimeoutMs > 0, "connection-timeout-ms must be greater than 0");
FETCHER.getParams().setIntParameter(HttpConnectionParams.CONNECTION_TIMEOUT, connectionTimeoutMs);
}
/**
* Change the global read timeout for all new fetchs.
*
* @param readTimeoutMs new connection timeout in milliseconds
*/
@Inject(optional = true)
public void setReadTimeoutMs(@Named("shindig.http.client.read-timeout-ms") int readTimeoutMs) {
Preconditions.checkArgument(readTimeoutMs > 0, "read-timeout-ms must be greater than 0");
FETCHER.getParams().setIntParameter(HttpConnectionParams.SO_TIMEOUT, readTimeoutMs);
}
/**
* @param response The response to parse
* @return A HttpResponse object made by consuming the response of the
* given HttpMethod.
* @throws IOException when problems occur processing the body content
*/
private HttpResponse makeResponse(org.apache.http.HttpResponse response) throws IOException {
HttpResponseBuilder builder = new HttpResponseBuilder();
if (response.getAllHeaders() != null) {
for (Header h : response.getAllHeaders()) {
if (h.getName() != null)
builder.addHeader(h.getName(), h.getValue());
}
}
HttpEntity entity = response.getEntity();
if (maxObjSize > 0 && entity.getContentLength() > maxObjSize) {
return HttpResponse.badrequest("Exceeded maximum number of bytes - " + maxObjSize);
}
byte[] responseBytes = (entity == null) ? null : toByteArraySafe(entity);
return builder
.setHttpStatusCode(response.getStatusLine().getStatusCode())
.setResponse(responseBytes)
.create();
}
/**
* This method is Safe replica version of org.apache.http.util.EntityUtils.toByteArray.
* The try block embedding 'instream.read' has a corresponding catch block for 'EOFException'
* (that's Ignored) and all other IOExceptions are let pass.
*
* @param entity
* @return byte array containing the entity content. May be empty/null.
* @throws IOException if an error occurs reading the input stream
*/
public byte[] toByteArraySafe(final HttpEntity entity) throws IOException {
if (entity == null) {
return null;
}
InputStream instream = entity.getContent();
if (instream == null) {
return new byte[] {};
}
Preconditions.checkArgument(entity.getContentLength() < Integer.MAX_VALUE, "HTTP entity too large to be buffered in memory");
// The raw data stream (inside JDK) is read in a buffer of size '512'. The original code
// org.apache.http.util.EntityUtils.toByteArray reads the unzipped data in a buffer of
// 4096 byte. For any data stream that has a compression ratio lesser than 1/8, this may
// result in the buffer/array overflow. Increasing the buffer size to '16384'. It's highly
// unlikely to get data compression ratios lesser than 1/32 (3%).
final int bufferLength = 16384;
int i = (int)entity.getContentLength();
if (i < 0) {
i = bufferLength;
}
ByteArrayBuffer buffer = new ByteArrayBuffer(i);
try {
byte[] tmp = new byte[bufferLength];
int l;
while((l = instream.read(tmp)) != -1) {
buffer.append(tmp, 0, l);
}
} catch (EOFException eofe) {
/**
* Ref: http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4040920
* Due to a bug in JDK ZLIB (InflaterInputStream), unexpected EOF error can occur.
* In such cases, even if the input stream is finished reading, the
* 'Inflater.finished()' call erroneously returns 'false' and
* 'java.util.zip.InflaterInputStream.fill' throws the 'EOFException'.
* So for such case, ignore the Exception in case Exception Cause is
* 'Unexpected end of ZLIB input stream'.
*
* Also, ignore this exception in case the exception has no message
* body as this is the case where {@link GZIPInputStream#readUByte}
* throws EOFException with empty message. A bug has been filed with Sun
* and will be mentioned here once it is accepted.
*/
if (instream.available() == 0 &&
(eofe.getMessage() == null ||
eofe.getMessage().equals("Unexpected end of ZLIB input stream"))) {
LOG.log(Level.FINE, "EOFException: ", eofe);
} else {
throw eofe;
}
}
finally {
instream.close();
}
return buffer.toByteArray();
}
}