/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;
import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
import org.apache.commons.httpclient.methods.multipart.FilePart;
import org.apache.commons.httpclient.methods.multipart.Part;
import org.apache.commons.httpclient.methods.multipart.PartSource;
import org.apache.commons.httpclient.methods.multipart.StringPart;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.*;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The {@link CommonsHttpSolrServer} uses the Apache Commons HTTP Client to connect to solr.
* <pre class="prettyprint" >SolrServer server = new CommonsHttpSolrServer( url );</pre>
*
* @version $Id: CommonsHttpSolrServer.java 1170589 2011-09-14 13:03:51Z janhoy $
* @since solr 1.3
*/
public class CommonsHttpSolrServer extends SolrServer
{
private static final long serialVersionUID = 1L;
/**
* User-Agent String as identified by the HTTP request by the {@link
* org.apache.commons.httpclient.HttpClient HttpClient} to the Solr
* server from the client.
*/
public static final String AGENT = "Solr["+CommonsHttpSolrServer.class.getName()+"] 1.0";
private static Logger log = LoggerFactory.getLogger(CommonsHttpSolrServer.class);
/**
* The URL of the Solr server.
*/
protected String _baseURL;
/**
* Default value: null / empty. <p/>
* Parameters that are added to every request regardless. This may be a place to add
* something like an authentication token.
*/
protected ModifiableSolrParams _invariantParams;
/**
* Default response parser is BinaryResponseParser <p/>
* This parser represents the default Response Parser chosen to
* parse the response if the parser were not specified as part of
* the request.
* @see org.apache.solr.client.solrj.impl.BinaryResponseParser
*/
protected ResponseParser _parser;
/**
* The RequestWriter used to write all requests to Solr
* @see org.apache.solr.client.solrj.request.RequestWriter
*/
protected RequestWriter requestWriter = new RequestWriter();
private final HttpClient _httpClient;
/**
* This defaults to false under the
* assumption that if you are following a redirect to get to a Solr
* installation, something is misconfigured somewhere.
*/
private boolean _followRedirects = false;
/**
* If compression is enabled, both gzip and deflate compression will
* be accepted in the HTTP response.
*/
private boolean _allowCompression = false;
/**
* Maximum number of retries to attempt in the event of transient
* errors. Default: 0 (no) retries. No more than 1 recommended.
*/
private int _maxRetries = 0;
/**
* Default value: <b> false </b>
* <p>
* If set to false, add the query parameters as URL-encoded parameters to the
* POST request in a single part. If set to true, create a new part of a
* multi-part request for each parameter.
*
* The reason for adding all parameters as parts of a multi-part request is
* that this allows us to specify the charset -- standards for single-part
* requests specify that non-ASCII characters should be URL-encoded, but don't
* specify the charset of the characters to be URL-encoded (cf.
* http://www.w3.org/TR/html401/interact/forms.html#form-content-type).
* Therefore you have to rely on your servlet container doing the right thing
* with single-part requests.
*/
private boolean useMultiPartPost;
/**
* @param solrServerUrl The URL of the Solr server. For
* example, "<code>http://localhost:8983/solr/</code>"
* if you are using the standard distribution Solr webapp
* on your local machine.
*/
public CommonsHttpSolrServer(String solrServerUrl) throws MalformedURLException {
this(new URL(solrServerUrl));
}
/**
* Talk to the Solr server via the given HttpClient. The connection manager
* for the client should be a MultiThreadedHttpConnectionManager if this
* client is being reused across SolrServer instances, or of multiple threads
* will use this SolrServer.
*/
public CommonsHttpSolrServer(String solrServerUrl, HttpClient httpClient) throws MalformedURLException {
this(new URL(solrServerUrl), httpClient, new BinaryResponseParser(), false);
}
public CommonsHttpSolrServer(String solrServerUrl, HttpClient httpClient, boolean useMultiPartPost) throws MalformedURLException {
this(new URL(solrServerUrl), httpClient, new BinaryResponseParser(), useMultiPartPost);
}
public CommonsHttpSolrServer(String solrServerUrl, HttpClient httpClient, ResponseParser parser) throws MalformedURLException {
this(new URL(solrServerUrl), httpClient, parser, false);
}
/**
* @param baseURL The URL of the Solr server. For example,
* "<code>http://localhost:8983/solr/</code>" if you are using the
* standard distribution Solr webapp on your local machine.
*/
public CommonsHttpSolrServer(URL baseURL)
{
this(baseURL, null, new BinaryResponseParser(), false);
}
public CommonsHttpSolrServer(URL baseURL, HttpClient client) {
this(baseURL, client, new BinaryResponseParser(), false);
}
/**
*
* @see #useMultiPartPost
*/
public CommonsHttpSolrServer(URL baseURL, HttpClient client, boolean useMultiPartPost) {
this(baseURL, client, new BinaryResponseParser(), useMultiPartPost);
}
/**
* @see #useMultiPartPost
* @see #_parser
*/
public CommonsHttpSolrServer(URL baseURL, HttpClient client, ResponseParser parser, boolean useMultiPartPost) {
_baseURL = baseURL.toExternalForm();
if( _baseURL.endsWith( "/" ) ) {
_baseURL = _baseURL.substring( 0, _baseURL.length()-1 );
}
if( _baseURL.indexOf( '?' ) >=0 ) {
throw new RuntimeException( "Invalid base url for solrj. The base URL must not contain parameters: "+_baseURL );
}
if (client == null) {
_httpClient = new HttpClient(new MultiThreadedHttpConnectionManager()) ;
// prevent retries (note: this didn't work when set on mgr.. needed to be set on client)
DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false);
_httpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler);
// set some better defaults if we created a new connection manager and client
// increase the default connections
this.setDefaultMaxConnectionsPerHost( 32 ); // 2
this.setMaxTotalConnections( 128 ); // 20
} else {
_httpClient = client;
}
_parser = parser;
this.useMultiPartPost = useMultiPartPost;
}
//------------------------------------------------------------------------
//------------------------------------------------------------------------
/**
* Process the request. If {@link org.apache.solr.client.solrj.SolrRequest#getResponseParser()} is null, then use
* {@link #getParser()}
* @param request The {@link org.apache.solr.client.solrj.SolrRequest} to process
* @return The {@link org.apache.solr.common.util.NamedList} result
* @throws SolrServerException
* @throws IOException
*
* @see #request(org.apache.solr.client.solrj.SolrRequest, org.apache.solr.client.solrj.ResponseParser)
*/
@Override
public NamedList<Object> request( final SolrRequest request ) throws SolrServerException, IOException
{
ResponseParser responseParser = request.getResponseParser();
if (responseParser == null) {
responseParser = _parser;
}
return request(request, responseParser);
}
public NamedList<Object> request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException {
HttpMethod method = null;
InputStream is = null;
SolrParams params = request.getParams();
Collection<ContentStream> streams = requestWriter.getContentStreams(request);
String path = requestWriter.getPath(request);
if( path == null || !path.startsWith( "/" ) ) {
path = "/select";
}
ResponseParser parser = request.getResponseParser();
if( parser == null ) {
parser = _parser;
}
// The parser 'wt=' and 'version=' params are used instead of the original params
ModifiableSolrParams wparams = new ModifiableSolrParams();
wparams.set( CommonParams.WT, parser.getWriterType() );
wparams.set( CommonParams.VERSION, parser.getVersion());
if( params == null ) {
params = wparams;
}
else {
params = new DefaultSolrParams( wparams, params );
}
if( _invariantParams != null ) {
params = new DefaultSolrParams( _invariantParams, params );
}
int tries = _maxRetries + 1;
try {
while( tries-- > 0 ) {
// Note: since we aren't do intermittent time keeping
// ourselves, the potential non-timeout latency could be as
// much as tries-times (plus scheduling effects) the given
// timeAllowed.
try {
if( SolrRequest.METHOD.GET == request.getMethod() ) {
if( streams != null ) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!" );
}
method = new GetMethod( _baseURL + path + ClientUtils.toQueryString( params, false ) );
}
else if( SolrRequest.METHOD.POST == request.getMethod() ) {
String url = _baseURL + path;
boolean isMultipart = ( streams != null && streams.size() > 1 );
if (streams == null || isMultipart) {
PostMethod post = new PostMethod(url);
post.getParams().setContentCharset("UTF-8");
if (!this.useMultiPartPost && !isMultipart) {
post.addRequestHeader("Content-Type","application/x-www-form-urlencoded; charset=UTF-8");
}
List<Part> parts = new LinkedList<Part>();
Iterator<String> iter = params.getParameterNamesIterator();
while (iter.hasNext()) {
String p = iter.next();
String[] vals = params.getParams(p);
if (vals != null) {
for (String v : vals) {
if (this.useMultiPartPost || isMultipart) {
parts.add(new StringPart(p, v, "UTF-8"));
} else {
post.addParameter(p, v);
}
}
}
}
if (isMultipart) {
int i = 0;
for (ContentStream content : streams) {
final ContentStream c = content;
String charSet = null;
PartSource source = new PartSource() {
public long getLength() {
return c.getSize();
}
public String getFileName() {
return c.getName();
}
public InputStream createInputStream() throws IOException {
return c.getStream();
}
};
parts.add(new FilePart(c.getName(), source,
c.getContentType(), charSet));
}
}
if (parts.size() > 0) {
post.setRequestEntity(new MultipartRequestEntity(parts
.toArray(new Part[parts.size()]), post.getParams()));
}
method = post;
}
// It is has one stream, it is the post body, put the params in the URL
else {
String pstr = ClientUtils.toQueryString(params, false);
PostMethod post = new PostMethod(url + pstr);
// post.setRequestHeader("connection", "close");
// Single stream as body
// Using a loop just to get the first one
final ContentStream[] contentStream = new ContentStream[1];
for (ContentStream content : streams) {
contentStream[0] = content;
break;
}
if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
post.setRequestEntity(new RequestEntity() {
public long getContentLength() {
return -1;
}
public String getContentType() {
return contentStream[0].getContentType();
}
public boolean isRepeatable() {
return false;
}
public void writeRequest(OutputStream outputStream) throws IOException {
((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream);
}
}
);
} else {
is = contentStream[0].getStream();
post.setRequestEntity(new InputStreamRequestEntity(is, contentStream[0].getContentType()));
}
method = post;
}
}
else {
throw new SolrServerException("Unsupported method: "+request.getMethod() );
}
}
catch( NoHttpResponseException r ) {
// This is generally safe to retry on
method.releaseConnection();
method = null;
if(is != null) {
is.close();
}
// If out of tries then just rethrow (as normal error).
if( ( tries < 1 ) ) {
throw r;
}
//log.warn( "Caught: " + r + ". Retrying..." );
}
}
}
catch( IOException ex ) {
log.error("####request####",ex);
throw new SolrServerException("error reading streams", ex );
}
method.setFollowRedirects( _followRedirects );
method.addRequestHeader( "User-Agent", AGENT );
if( _allowCompression ) {
method.setRequestHeader( new Header( "Accept-Encoding", "gzip,deflate" ) );
}
// method.setRequestHeader("connection", "close");
try {
// Execute the method.
//System.out.println( "EXECUTE:"+method.getURI() );
int statusCode = _httpClient.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
StringBuilder msg = new StringBuilder();
msg.append( method.getStatusLine().getReasonPhrase() );
msg.append( "\n\n" );
msg.append( method.getStatusText() );
msg.append( "\n\n" );
msg.append( "request: "+method.getURI() );
throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8") );
}
// Read the contents
String charset = "UTF-8";
if( method instanceof HttpMethodBase ) {
charset = ((HttpMethodBase)method).getResponseCharSet();
}
InputStream respBody = method.getResponseBodyAsStream();
// Jakarta Commons HTTPClient doesn't handle any
// compression natively. Handle gzip or deflate
// here if applicable.
if( _allowCompression ) {
Header contentEncodingHeader = method.getResponseHeader( "Content-Encoding" );
if( contentEncodingHeader != null ) {
String contentEncoding = contentEncodingHeader.getValue();
if( contentEncoding.contains( "gzip" ) ) {
//log.debug( "wrapping response in GZIPInputStream" );
respBody = new GZIPInputStream( respBody );
}
else if( contentEncoding.contains( "deflate" ) ) {
//log.debug( "wrapping response in InflaterInputStream" );
respBody = new InflaterInputStream(respBody);
}
}
else {
Header contentTypeHeader = method.getResponseHeader( "Content-Type" );
if( contentTypeHeader != null ) {
String contentType = contentTypeHeader.getValue();
if( contentType != null ) {
if( contentType.startsWith( "application/x-gzip-compressed" ) ) {
//log.debug( "wrapping response in GZIPInputStream" );
respBody = new GZIPInputStream( respBody );
}
else if ( contentType.startsWith("application/x-deflate") ) {
//log.debug( "wrapping response in InflaterInputStream" );
respBody = new InflaterInputStream(respBody);
}
}
}
}
}
return processor.processResponse(respBody, charset);
}
catch (HttpException e) {
throw new SolrServerException( e );
}
catch (IOException e) {
throw new SolrServerException( e );
}
finally {
method.releaseConnection();
if(is != null) {
is.close();
}
}
}
//-------------------------------------------------------------------
//-------------------------------------------------------------------
/**
* Retrieve the default list of parameters are added to every request regardless.
*
* @see #_invariantParams
*/
public ModifiableSolrParams getInvariantParams()
{
return _invariantParams;
}
public String getBaseURL() {
return _baseURL;
}
public void setBaseURL(String baseURL) {
this._baseURL = baseURL;
}
public ResponseParser getParser() {
return _parser;
}
/**
* Note: This setter method is <b>not thread-safe</b>.
* @param processor Default Response Parser chosen to parse the response if the parser were not specified as part of the request.
* @see org.apache.solr.client.solrj.SolrRequest#getResponseParser()
*/
public void setParser(ResponseParser processor) {
_parser = processor;
}
public HttpClient getHttpClient() {
return _httpClient;
}
private HttpConnectionManager getConnectionManager() {
return _httpClient.getHttpConnectionManager();
}
/** set connectionTimeout on the underlying HttpConnectionManager
* @param timeout Timeout in milliseconds
**/
public void setConnectionTimeout(int timeout) {
getConnectionManager().getParams().setConnectionTimeout(timeout);
}
/** set connectionManagerTimeout on the HttpClient.
* @param timeout Timeout in milliseconds
* @deprecated Use {@link #setConnectionManagerTimeout(long)} **/
@Deprecated
public void setConnectionManagerTimeout(int timeout) {
_httpClient.getParams().setConnectionManagerTimeout(timeout);
}
/**
* Sets soTimeout (read timeout) on the underlying
* HttpConnectionManager. This is desirable for queries, but
* probably not for indexing.
*
* @param timeout Timeout in milliseconds
*/
public void setConnectionManagerTimeout(long timeout) {
_httpClient.getParams().setConnectionManagerTimeout(timeout);
}
/**
* Sets soTimeout (read timeout) on the underlying
* HttpConnectionManager. This is desirable for queries, but
* probably not for indexing.
* @param timeout Timeout in milliseconds
**/
public void setSoTimeout(int timeout) {
getConnectionManager().getParams().setSoTimeout(timeout);
}
/** set maxConnectionsPerHost on the underlying HttpConnectionManager */
public void setDefaultMaxConnectionsPerHost(int connections) {
getConnectionManager().getParams().setDefaultMaxConnectionsPerHost(connections);
}
/** set maxTotalConnection on the underlying HttpConnectionManager */
public void setMaxTotalConnections(int connections) {
getConnectionManager().getParams().setMaxTotalConnections(connections);
}
/**
* set followRedirects.
* @see #_followRedirects
*/
public void setFollowRedirects( boolean followRedirects ) {
_followRedirects = followRedirects;
}
/**
* set allowCompression.
* @see #_allowCompression
*/
public void setAllowCompression( boolean allowCompression ) {
_allowCompression = allowCompression;
}
/**
* set maximum number of retries to attempt in the event of
* transient errors.
* @param maxRetries No more than 1 recommended
* @see #_maxRetries
*/
public void setMaxRetries( int maxRetries ) {
// if (maxRetries > 1) {
// log.warn("CommonsHttpSolrServer: maximum Retries " + maxRetries + " > 1. Maximum recommended retries is 1.");
// }
_maxRetries = maxRetries;
}
public void setRequestWriter(RequestWriter requestWriter) {
this.requestWriter = requestWriter;
}
/**
* Adds the documents supplied by the given iterator.
* @param docIterator the iterator which returns SolrInputDocument instances
* @return the response from the SolrServer
*/
public UpdateResponse add(Iterator<SolrInputDocument> docIterator)
throws SolrServerException, IOException {
return add(docIterator, -1);
}
/**
* Adds the documents supplied by the given iterator, specifying max time before they become committed
* @param docIterator the iterator which returns SolrInputDocument instances
* @param commitWithinMs the time in milliseconds before a commit automatically is triggered
* @return the response from the SolrServer
*/
public UpdateResponse add(Iterator<SolrInputDocument> docIterator, int commitWithinMs)
throws SolrServerException, IOException {
UpdateRequest req = new UpdateRequest();
req.setDocIterator(docIterator);
req.setCommitWithin(commitWithinMs);
return req.process(this);
}
/**
* Adds the beans supplied by the given iterator
* @param beanIterator the iterator which returns Beans
* @return the response from the SolrServer
*/
public UpdateResponse addBeans(final Iterator<?> beanIterator)
throws SolrServerException, IOException {
return addBeans(beanIterator, -1);
}
/**
* Adds the beans supplied by the given iterator, specifying max time before they become committed
* @param commitWithinMs the time in milliseconds before a commit automatically is triggered
* @param beanIterator the iterator which returns Beans
* @return the response from the SolrServer
*/
public UpdateResponse addBeans(final Iterator<?> beanIterator, int commitWithinMs)
throws SolrServerException, IOException {
UpdateRequest req = new UpdateRequest();
req.setDocIterator(new Iterator<SolrInputDocument>() {
public boolean hasNext() {
return beanIterator.hasNext();
}
public SolrInputDocument next() {
Object o = beanIterator.next();
if (o == null) return null;
return getBinder().toSolrInputDocument(o);
}
public void remove() {
beanIterator.remove();
}
});
req.setCommitWithin(commitWithinMs);
return req.process(this);
}
}