/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.MalformedURLException;
import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.*;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.NamedList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* {@link StreamingUpdateSolrServer} buffers all added documents and writes them
* into open HTTP connections. This class is thread safe.
*
* Although any SolrServer request can be made with this implementation,
* it is only recommended to use the {@link StreamingUpdateSolrServer} with
* /update requests. The query interface is better suited for
*
* @version $Id: CommonsHttpSolrServer.java 724175 2008-12-07 19:07:11Z ryan $
* @since solr 1.4
*/
public class StreamingUpdateSolrServer extends CommonsHttpSolrServer
{
static final Logger log = LoggerFactory.getLogger( StreamingUpdateSolrServer.class );
final BlockingQueue<UpdateRequest> queue;
final ExecutorService scheduler = Executors.newCachedThreadPool();
final String updateUrl = "/update";
final Queue<Runner> runners;
volatile CountDownLatch lock = null; // used to block everything
final int threadCount;
/**
* Uses an internal MultiThreadedHttpConnectionManager to manage http connections
*
* @param solrServerUrl The Solr server URL
* @param queueSize The buffer size before the documents are sent to the server
* @param threadCount The number of background threads used to empty the queue
* @throws MalformedURLException
*/
public StreamingUpdateSolrServer(String solrServerUrl, int queueSize, int threadCount) throws MalformedURLException {
this(solrServerUrl, null, queueSize, threadCount);
}
/**
* Uses the supplied HttpClient to send documents to the Solr server, the HttpClient should be instantiated using a
* MultiThreadedHttpConnectionManager.
*/
public StreamingUpdateSolrServer(String solrServerUrl, HttpClient client, int queueSize, int threadCount) throws MalformedURLException {
super(solrServerUrl, client);
queue = new LinkedBlockingQueue<UpdateRequest>(queueSize);
this.threadCount = threadCount;
runners = new LinkedList<Runner>();
}
/**
* Opens a connection and sends everything...
*/
class Runner implements Runnable {
final Lock runnerLock = new ReentrantLock();
public void run() {
runnerLock.lock();
// info is ok since this should only happen once for each thread
log.info( "starting runner: {}" , this );
PostMethod method = null;
try {
do {
try {
RequestEntity request = new RequestEntity() {
// we don't know the length
public long getContentLength() { return -1; }
public String getContentType() { return ClientUtils.TEXT_XML; }
public boolean isRepeatable() { return false; }
public void writeRequest(OutputStream out) throws IOException {
try {
OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8");
writer.append( "<stream>" ); // can be anything...
UpdateRequest req = queue.poll( 250, TimeUnit.MILLISECONDS );
while( req != null ) {
log.debug( "sending: {}" , req );
req.writeXML( writer );
// check for commit or optimize
SolrParams params = req.getParams();
if( params != null ) {
String fmt = null;
if( params.getBool( UpdateParams.OPTIMIZE, false ) ) {
fmt = "<optimize waitSearcher=\"%s\" waitFlush=\"%s\" />";
}
else if( params.getBool( UpdateParams.COMMIT, false ) ) {
fmt = "<commit waitSearcher=\"%s\" waitFlush=\"%s\" />";
}
if( fmt != null ) {
log.info( fmt );
writer.write( String.format( fmt,
params.getBool( UpdateParams.WAIT_SEARCHER, false )+"",
params.getBool( UpdateParams.WAIT_FLUSH, false )+"") );
}
}
writer.flush();
req = queue.poll( 250, TimeUnit.MILLISECONDS );
}
writer.append( "</stream>" );
writer.flush();
}
catch (InterruptedException e) {
e.printStackTrace();
}
}
};
method = new PostMethod(_baseURL+updateUrl );
method.setRequestEntity( request );
method.setFollowRedirects( false );
method.addRequestHeader( "User-Agent", AGENT );
int statusCode = getHttpClient().executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
StringBuilder msg = new StringBuilder();
msg.append( method.getStatusLine().getReasonPhrase() );
msg.append( "\n\n" );
msg.append( method.getStatusText() );
msg.append( "\n\n" );
msg.append( "request: "+method.getURI() );
handleError( new Exception( msg.toString() ) );
}
} finally {
try {
// make sure to release the connection
if(method != null)
method.releaseConnection();
}
catch( Exception ex ){}
}
} while( ! queue.isEmpty());
}
catch (Throwable e) {
handleError( e );
}
finally {
// remove it from the list of running things unless we are the last runner and the queue is full...
// in which case, the next queue.put() would block and there would be no runners to handle it.
// This case has been further handled by using offer instead of put, and using a retry loop
// to avoid blocking forever (see request()).
synchronized (runners) {
if (runners.size() == 1 && queue.remainingCapacity() == 0) {
// keep this runner alive
scheduler.execute(this);
} else {
runners.remove( this );
}
}
log.info( "finished: {}" , this );
runnerLock.unlock();
}
}
}
@Override
public NamedList<Object> request( final SolrRequest request ) throws SolrServerException, IOException
{
if( !(request instanceof UpdateRequest) ) {
return super.request( request );
}
UpdateRequest req = (UpdateRequest)request;
// this happens for commit...
if( req.getDocuments()==null || req.getDocuments().isEmpty() ) {
blockUntilFinished();
return super.request( request );
}
SolrParams params = req.getParams();
if( params != null ) {
// check if it is waiting for the searcher
if( params.getBool( UpdateParams.WAIT_SEARCHER, false ) ) {
log.info( "blocking for commit/optimize" );
blockUntilFinished(); // empty the queue
return super.request( request );
}
}
try {
CountDownLatch tmpLock = lock;
if( tmpLock != null ) {
tmpLock.await();
}
boolean success = queue.offer(req);
for(;;) {
synchronized( runners ) {
if( runners.isEmpty()
|| (queue.remainingCapacity() < queue.size() // queue is half full and we can add more runners
&& runners.size() < threadCount) )
{
// We need more runners, so start a new one.
Runner r = new Runner();
runners.add( r );
scheduler.execute( r );
} else {
// break out of the retry loop if we added the element to the queue successfully, *and*
// while we are still holding the runners lock to prevent race conditions.
// race conditions.
if (success) break;
}
}
// Retry to add to the queue w/o the runners lock held (else we risk temporary deadlock)
// This retry could also fail because
// 1) existing runners were not able to take off any new elements in the queue
// 2) the queue was filled back up since our last try
// If we succeed, the queue may have been completely emptied, and all runners stopped.
// In all cases, we should loop back to the top to see if we need to start more runners.
//
if (!success) {
success = queue.offer(req, 100, TimeUnit.MILLISECONDS);
}
}
}
catch (InterruptedException e) {
log.error( "interrupted", e );
throw new IOException( e.getLocalizedMessage() );
}
// RETURN A DUMMY result
NamedList<Object> dummy = new NamedList<Object>();
dummy.add( "NOTE", "the request is processed in a background stream" );
return dummy;
}
public synchronized void blockUntilFinished()
{
lock = new CountDownLatch(1);
try {
// Wait until no runners are running
for(;;) {
Runner runner;
synchronized(runners) {
runner = runners.peek();
}
if (runner == null) break;
runner.runnerLock.lock();
runner.runnerLock.unlock();
}
} finally {
lock.countDown();
lock=null;
}
}
public void handleError( Throwable ex )
{
log.error( "error", ex );
}
}