package com.caringo.client.request;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
import java.io.ByteArrayOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpConnectionManager;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.CircularRedirectException;
import org.apache.commons.httpclient.RedirectException;
import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import com.caringo.client.ScspExecutionException;
import com.caringo.client.ScspHeaders;
import com.caringo.client.ScspResponseOutputStream;
/**
*
* Class: ScspRequestHandler <br>
* Package: com.caringo.client<br>
* The ScspClient class is the API for the client's communication with CAStor. It exposes functions like
* "write, update, read, info, delete". <br>
* Copyright (c) 2008 by Caringo, Inc. -- All rights reserved<br>
* This is free software, distributed under the terms of the New BSD license.<br>
* See the LICENSE.txt file included in this archive.<br>
* <br>
*
* @author jmike
* @created Jul 21, 2008
*/
public class ScspRequestHandler {
private static final int BUFFER_LEN = 32768;
private HttpClient client;
private LocatorRedirectHandler locator;
private int maxRetries;
private int retries;
private int externalTimeout;
private int currentTimeout;
private HttpConnectionManager externalConnMgr;
private String authenticationNonce;
private String authenticationRealm;
private String userName;
private String password;
// /
// / Construction
// /
/**
* @param client
* the http client
* @param locator
* the locator
* @param maxRetries
* how many times to try to accomplish a request
* @param externalTimeout -
* Idle timeout for requests resulting from a 305, in seconds.
* @param externalConnMgr -
* Connection manager for requests resulting from a 305
*/
public ScspRequestHandler(HttpClient client, LocatorRedirectHandler locator, int maxRetries, int externalTimeout, HttpConnectionManager externalConnMgr) {
this.locator = locator;
this.client = client;
this.maxRetries = maxRetries;
this.retries = 0;
this.userName = "";
this.password = "";
this.authenticationNonce = "";
this.authenticationRealm = "";
this.externalTimeout = externalTimeout * 1000;
this.externalConnMgr = externalConnMgr;
this.currentTimeout = this.client.getParams().getSoTimeout();
}
public void setAuthentication(String userName, String password, String nonce, String realm) {
this.userName = userName;
this.password = password;
this.authenticationNonce = nonce;
this.authenticationRealm = realm;
}
/**
* This method will write the specified stream to castor, and send back the results in the handler object.
*
* @param path
* - request path or null if none.
* @param stream
* - the inputstream to be written
* @param streamLength
* - the length of the stream
* @param contentType
* - the content type
* @param headers
* - the headers
* @param queryArgs
* - array of NameValuePair objects to be passed as queryArguments to the HTTP method.
* @param handler
* - the response output stream handler
* @throws ScspExecutionException
* @throws InvalidPolicyException
*/
public void write(String path, InputStream stream, long streamLength, String contentType, ScspHeaders headers,
String queryArgs, ScspResponseOutputStream handler) throws ScspExecutionException {
PostMethodFactory methodFactory = new PostMethodFactory(stream, streamLength, contentType, headers);
executeMethodWithRetries(path, handler, methodFactory, queryArgs);
}
/**
* This method will update an anchor stream with the new stream and metadata provided.
*
* @param path
* - request path or null if none.
* @param stream
* - the updated stream
* @param streamLength
* the updated stream's length.
* @param contentType
* - the updated stream's content type
* @param headers
* - the updated stream's new headers
* @param queryArgs
* - the updated stream's new query args.
* @param handler
* - the scsp response handler.
* @throws ScspExecutionException
*/
public void update(String path, InputStream stream, long streamLength, String contentType, ScspHeaders headers,
String queryArgs, ScspResponseOutputStream handler) throws ScspExecutionException {
PutMethodFactory methodFactory = new PutMethodFactory(stream, streamLength, contentType, headers);
executeMethodWithRetries(path, handler, methodFactory, queryArgs);
}
/**
* this method will read the specified uuid, and return the results in the handler.
*
* @param path
* - request path or null if none.
* @param headers
* - read query headers
* @param queryArgs
* - array of NameValuePair objects to be passed as queryArguments to the HTTP method.
* @param handler
* - the response handler
* @throws ScspExecutionException
*/
public void read(String path, ScspHeaders headers, String queryArgs, ScspResponseOutputStream handler) throws ScspExecutionException {
executeMethodWithRetries(path, handler, new GetMethodFactory(headers), queryArgs);
}
/**
* this method will info/head the specified uuid, and return the results in the handler.
*
* @param path
* - request path or null if none.
* @param headers
* - read query headers
* @param queryArgs
* - array of NameValuePair objects to be passed as queryArguments to the HTTP method.
* @param handler
* - the response handler
* @throws ScspExecutionException
*/
public void info(String path, ScspHeaders headers, String queryArgs, ScspResponseOutputStream handler) throws ScspExecutionException {
executeMethodWithRetries(path, handler, new HeadMethodFactory(headers), queryArgs);
}
/**
* this method will delete the specified uuid, and return the results in the handler.
*
* @param path
* - request path or null if none.
* @param headers
* - read query headers
* @param queryArgs
* - array of NameValuePair objects to be passed as queryArguments to the HTTP method.
* @param handler
* - the response handler
* @throws ScspExecutionException
*/
public void delete(String path, ScspHeaders headers, String queryArgs, ScspResponseOutputStream handler) throws ScspExecutionException {
executeMethodWithRetries(path, handler, new DeleteMethodFactory(headers), queryArgs);
}
/**
* this method will copy the specified path, and return the results in the handler.
*
* @param path
* - request path or null if none.
* @param headers
* - read query headers
* @param queryArgs
* - array of NameValuePair objects to be passed as queryArguments to the HTTP method.
* @param handler
* - the response handler
* @throws ScspExecutionException
*/
public void copy(String path, ScspHeaders headers, String queryArgs, ScspResponseOutputStream handler) throws ScspExecutionException {
executeMethodWithRetries(path, handler, new CopyMethodFactory(headers), queryArgs);
}
/**
* this method will append the input stream to the specified uuid, and return the results in the handler.
*
* @param path
* - request path or null if none.
* @param stream
* - the updated stream
* @param streamLength
* - the updated stream's length.
* @param contentType
* - the updated stream's content type
* @param headers
* - read query headers
* @param queryArgs
* - array of NameValuePair objects to be passed as queryArguments to the HTTP method.
* @param handler
* - the response handler
* @throws ScspExecutionException
*/
public void append(String path, InputStream stream, long streamLength, String contentType, ScspHeaders headers,
String queryArgs, ScspResponseOutputStream handler) throws ScspExecutionException {
AppendMethodFactory methodFactory = new AppendMethodFactory(stream, streamLength, contentType, headers);
executeMethodWithRetries(path, handler, methodFactory, queryArgs);
}
/**
* Set the user agent to pass in HTTP requests to CAStor.
*
* @param userAgent -
* the user agent to pass to CAStor.
*/
public void setUserAgent(String userAgent) {
this.client.getParams().setParameter(HttpClientParams.USER_AGENT, userAgent);
}
/**
* Get the user agent.
*
* @return the user agent.
*/
public String getUserAgent() {
return this.client.getParams().getParameter(HttpClientParams.USER_AGENT).toString();
}
/**
* Set the idle timeout for requests resulting from a 305, in seconds.
*
* @param externalTimeout -
* the post-305 idle timeout
*/
public void setExternalTimeout(int externalTimeout) {
this.externalTimeout = externalTimeout * 1000;
}
/**
* Get idle timeout for requests resulting from a 305, in seconds.
*
* @return the post-305 idle timeout
*/
public int getExternalTimeout() {
return this.externalTimeout / 1000;
}
// /
// / Support
// /
/**
* Execute HTTP methods, with retries.
*
* @param path
* request path "" or null if none.
* @param handler
* the ScspResponseHandler
* @param methodFactory
* factory that will create the method to be executed.
* @param queryArgs
* - array of NameValuePair objects to be passed as queryArguments to the HTTP method.
*/
private void executeMethodWithRetries(String path, ScspResponseOutputStream handler, MethodFactory methodFactory,
String queryArgs) throws ScspExecutionException {
URI uri = null;
this.retries = 0;
boolean handled = false;
Exception savedException = null;
while (!handled) {
if (this.retries > maxRetries) {
if (savedException == null) {
throw new ScspExecutionException("Too many SCSP retries.");
}
else {
throw new ScspExecutionException("Too many SCSP retries.", savedException);
}
}
try {
savedException = null;
uri = getUri(path);
// only append query args if there are args.
if (queryArgs != null && queryArgs.length() > 0) {
String qString = queryArgs;
uri.setQuery(qString);
}
// PMR FIX TODO
// Handle 500 status
executeMethod(handler, methodFactory, uri, queryArgs);
handled = true;
} catch (InterruptedIOException ioex) {
// this should be the exception that comes up for a request timeout
savedException = ioex;
try {
Thread.sleep(maxRetries * client.getParams().getIntParameter(HttpClientParams.SO_TIMEOUT, 10000) / 2);
}
catch(InterruptedException ex) {
// ignore
}
this.retries++;
} catch (IOException ex) {
savedException = ex;
uri = null;
this.retries++;
}
}
}
private void setAuthentication(HttpMethod method) {
if (userName.length() > 0 && password.length() > 0 && authenticationRealm.length() > 0 ) {
// Commons HttpClient 3 doesn't support reusing the server nonce or preemptive digest authentication, so we're ignoring that.
// If we use HttpComponents in the future, then we can use its features for preemptive authentication
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(userName, password);
method.setDoAuthentication(true);
// We don't care about the host or port, so let the be anything.
client.getState().setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, authenticationRealm), credentials);
}
else {
method.setDoAuthentication(false);
}
}
/**
* @param handler
* @param methodFactory
* @param queryArgs
* @throws IOException
* @throws HttpException
* @throws ScspExecutionException
*/
private void executeMethod(ScspResponseOutputStream handler, MethodFactory methodFactory, URI uri, String queryArgs)
throws ScspExecutionException, HttpException, IOException {
URI executeURI;
try {
executeURI = (URI)uri.clone();
} catch (java.lang.CloneNotSupportedException e) {
throw new ScspExecutionException("No URI.clone: " + e);
}
ArrayList<String> authsWeTried = new ArrayList<String>();
authsWeTried.add(uri.getAuthority());
int redirects = 0;
Boolean handled = false;
HttpMethod method = null;
try {
while (!handled)
{
method = methodFactory.makeMethod();
method.setURI(executeURI);
method.setFollowRedirects(false);
setAuthentication(method);
client.executeMethod(method);
int statusCode = method.getStatusCode();
if ((statusCode == 301) || (statusCode == 305) || (statusCode == 307))
{
Header redirectHeader = method.getResponseHeader("Location");
String redirectValue = redirectHeader.getValue();
// to get the auth arg and the authority. Otherwise, we want everything off the old URI
URI redirectURI = new URI(redirectValue, true); // assume it's escaped
String redirectAuthority = redirectURI.getAuthority();
if (authsWeTried.contains(redirectAuthority))
{
// We're in a loop.
throw new CircularRedirectException(); // Throw unless we want to start over.
}
authsWeTried.add(redirectAuthority);
redirects++;
// See if the redirect limit is set on the client. If not, use 10.
if (redirects > client.getParams().getIntParameter(HttpClientParams.MAX_REDIRECTS, 10))
{
// Too many redirects. Something bad happened.
throw new RedirectException("Too many redirects");
}
HttpConnectionManager newConnMgr = null;
if (statusCode == 301) {
// Moved permanently, so notify the locator
notifyRedirect(executeURI, redirectURI);
} else if (statusCode == 305) {
newConnMgr = this.externalConnMgr;
client.getParams().setParameter(HttpClientParams.SO_TIMEOUT, this.externalTimeout);
this.currentTimeout = this.externalTimeout;
}
//else it was a 307 moved temporarily, so just reissue without notifying
method.releaseConnection();
if (null != newConnMgr) {
client.setHttpConnectionManager(newConnMgr);
}
executeURI = redirectURI;
continue; // go back and do it again;
}
else
{
handled = true;
}
sendPositiveLocatorFeedback(uri);
// HEAD content-length doesnt refer to the body length, 'cause
// HTTP's goofy
// Also, an if-modified header can come back with a 304 (not modified) or 402 (precondition failed)
// with an empty body but a non-zero content length.
long bodyLength = ("HEAD".equals(method.getName()) ||
method.getStatusCode() == 304 ||
method.getStatusCode() == 402) ? 0 : getResponseContentLength(method);
boolean wantsBody = handler.responseReceived(method.getStatusCode(), method.getResponseHeaders(),
method.getStatusLine().toString(), bodyLength);
// for chunked encoding, we don't have a body length
if (bodyLength > 0 || (method.getResponseHeader("transfer-encoding") != null &&
method.getResponseHeader("transfer-encoding").getValue().compareToIgnoreCase("chunked") == 0)) {
if (wantsBody) {
pipeResponseToHandler(method, handler);
} else {
method.getResponseBodyAsStream().close();
}
// HttpClient won't have the footer/trailer headers until the body is read.
if (method.getResponseHeader("transfer-encoding") != null) {
handler.footersReceived(method.getResponseFooters());
}
}
handler.responseComplete(this.retries);
}
} catch (HttpException ex) {
throw ex;
} catch (IOException ex) {
sendNegativeLocatorFeedback(uri);
throw ex;
} finally {
if (method != null) {
method.releaseConnection();
}
}
}
/**
* @param method
* @param handler
* @throws ScspExecutionException
* Note that this can hang the entire interface if the handler blocks, so be very careful to return quickly from this
*/
private void pipeResponseToHandler(HttpMethod method, ScspResponseOutputStream handler) throws ScspExecutionException {
long byteCount = 0;
int maxTimeout = this.currentTimeout;
final int DELAY_PER_RETRY = 100; // milliseconds
final int MAX_RETRIES = (maxTimeout / DELAY_PER_RETRY) + 1;
boolean unlimitedRetries = (maxTimeout == 0);
try {
byte[] buf = new byte[BUFFER_LEN];
int read = 0;
//If we performance becomes an issue, then we can change the body handling to callback with the response body stream from
//the method. Also, see the Request Streaming section of the HttpClient performance guide at
//http://hc.apache.org/httpclient-3.x/performance.html.
InputStream in = method.getResponseBodyAsStream();
if (in != null) { // fix for 5179. Some requests, notably a get with a if-{un}modified-since... with a non-qualifying date, can return an empty body.
int retries = MAX_RETRIES;
while (retries > 0) {
try {
read = in.read(buf);
while (read != -1) {
byteCount += read;
handler.bodyDataReceived(buf, 0, read);
retries = MAX_RETRIES; // we got a success, so reset the retry count
read = in.read(buf);
}
break;
} catch (SocketTimeoutException ex) {
// Don't worry about byte count. It is possible that network delays could cause this to happen mid-
// stream, so just keep retrying until all retries are exhausted
if (!unlimitedRetries) {
retries -= 1;
}
if (retries > 0) {
try {
Thread.sleep(DELAY_PER_RETRY);
} catch (InterruptedException iex) {} // ignore
} else {
// Too many retries, so we've met the total socket timeout
throw ex;
}
}
}
}
} catch (IOException ex) {
// We need to throw a fatal error from here, because at this point,
// we
// don't know if the ultimate consumer of the body is still valid,
// so
// attempts to retry a GET, for example, might result in repeated
// data
// being written to the consumer.
throw new ScspExecutionException("Error writing to stream consumer", ex);
}
}
/**
* Get the response content-length value
*
* @param method
* @return
*/
private long getResponseContentLength(HttpMethod method) {
long result = 0;
Header contentLengthHdr = method.getResponseHeader("content-length");
if (contentLengthHdr != null) {
result = Long.parseLong(contentLengthHdr.getValue());
}
return result;
}
private URI getUri(String path) throws IOException {
if (path != null && !path.startsWith("/")) {
path = "/" + path; // URI class doesn't make sure the '/' separator is in place.
}
InetSocketAddress nodeAddr = locator.locate();
if (nodeAddr == null) {
throw new IOException("Unable to locate cluster node with locator " + locator);
}
InetAddress nodeInetAddr = nodeAddr.getAddress();
if (nodeInetAddr == null) {
locator.foundDead(nodeAddr);
throw new IOException("Unable to resolve cluster node name \"" + nodeAddr.getHostName() + "\" to IP address");
}
URI result = null;
try {
result = new URI("http", null, nodeInetAddr.getHostAddress(), nodeAddr.getPort());
result.setEscapedPath(pathEscape(path, null));
} catch (UnsupportedEncodingException e) {
throw new IOException("Unable to process request path: " + e.toString());
}
return result;
}
protected static final byte[] HEX_CHAR_TABLE = {
(byte)'0', (byte)'1', (byte)'2', (byte)'3',
(byte)'4', (byte)'5', (byte)'6', (byte)'7',
(byte)'8', (byte)'9', (byte)'a', (byte)'b',
(byte)'c', (byte)'d', (byte)'e', (byte)'f'
};
protected static final byte ESCAPE_CHAR = (byte)'%';
/**
* This should ONLY be used for the PATH portion of a URI!
*/
public static String pathEscape(String path, ArrayList<Byte> safeBytes) throws UnsupportedEncodingException {
if (null == path) {
return "";
}
byte[] utf8Bytes = path.getBytes("UTF-8");
ByteArrayOutputStream escapedBytes = new ByteArrayOutputStream();
for (byte c : utf8Bytes) {
if ((((byte)'0' <= c) && ((byte)'9' >= c)) ||
(((byte)'A' <= c) && ((byte)'Z' >= c)) ||
(((byte)'a' <= c) && ((byte)'z' >= c)) ||
((byte)'-' == c) ||
((byte)'_' == c) ||
((byte)'.' == c) ||
((byte)'!' == c) ||
((byte)'~' == c) ||
((byte)'*' == c) ||
((byte)'\'' == c) ||
((byte)'(' == c) ||
((byte)')' == c) ||
((byte)'/' == c) ||
((null != safeBytes) && safeBytes.contains(c))) {
// this character is safe to leave un-escaped
escapedBytes.write(c);
}
else {
// need to %-escape the byte
escapedBytes.write(ESCAPE_CHAR);
int v = c & 0xFF;
escapedBytes.write(HEX_CHAR_TABLE[v >>> 4]);
escapedBytes.write(HEX_CHAR_TABLE[v & 0x0F]);
}
}
return new String(escapedBytes.toByteArray(), "US-ASCII");
}
/**
* Feedback method, to let the locator know about a good URI.
*/
private void sendPositiveLocatorFeedback(URI uri) {
if (uri == null)
return;
try {
InetSocketAddress socketAddr = socketAddressFromUri(uri);
if (uri != null) {
locator.foundAlive(socketAddr);
}
} catch (URIException ex) {
// Ignore. It's an internal error, and we can just not tell the locator.
}
}
private void notifyRedirect(URI original, URI redirect) {
try {
locator.notifyRedirect(socketAddressFromUri(original), socketAddressFromUri(redirect));
} catch (URIException ex) {
// ignore the redirect.
}
}
/**
* Feedback method, to let the locator know the last URI was bad.
*/
private void sendNegativeLocatorFeedback(URI uri) {
if (uri == null)
return;
InetSocketAddress socketAddr;
try {
socketAddr = socketAddressFromUri(uri);
if (socketAddr != null) {
locator.foundDead(socketAddr);
}
} catch (URIException ex) {
// Ignore. It's an internal error, and we can just not tell the locator.
}
}
/**
* @param uri
* @return
* @throws URIException
*/
private InetSocketAddress socketAddressFromUri(URI uri) throws URIException {
InetSocketAddress result = null;
if (uri != null) {
String host = uri.getHost();
int port = uri.getPort();
if (port < 1) {
port = 80;
}
result = new InetSocketAddress(host, port);
}
return result;
}
}