LOGGER.fine("Result set from connector " + connectorName
+ " is NULL, no documents returned for traversal.");
return new BatchResult(TraversalDelayPolicy.POLL, 0);
}
Pusher pusher = null;
BatchResult result = null;
int counter = 0;
try {
// Get a Pusher for feeding the returned Documents.
pusher = pusherFactory.newPusher(connectorName);
while (true) {
if (Thread.currentThread().isInterrupted() || isCancelled()) {
LOGGER.fine("Traversal for connector " + connectorName
+ " has been interrupted; breaking out of batch run.");
break;
}
if (clock.getTimeMillis() >= timeoutTime) {
LOGGER.fine("Traversal batch for connector " + connectorName
+ " is completing due to time limit.");
break;
}
String docid = null;
try {
LOGGER.finer("Pulling next document from connector " + connectorName);
Document nextDocument = resultSet.nextDocument();
if (nextDocument == null) {
LOGGER.finer("Traversal batch for connector " + connectorName
+ " at end after processing " + counter + " documents.");
break;
} else {
// Since there are a couple of places below that could throw
// exceptions but not exit the while loop, the counter should be
// incremented here to insure it represents documents returned from
// the list. Note the call to nextDocument() could also throw a
// RepositoryDocumentException signaling a skipped document in which
// case the call will not be counted against the batch maximum.
counter++;
// Fetch DocId to use in messages.
try {
docid = Value.getSingleValueString(nextDocument,
SpiConstants.PROPNAME_DOCID);
} catch (IllegalArgumentException e1) {
LOGGER.finer("Unable to get document id for document ("
+ nextDocument + "): " + e1.getMessage());
} catch (RepositoryException e1) {
LOGGER.finer("Unable to get document id for document ("
+ nextDocument + "): " + e1.getMessage());
}
}
LOGGER.finer("Sending document (" + docid + ") from connector "
+ connectorName + " to Pusher");
if (pusher.take(nextDocument) != PusherStatus.OK) {
LOGGER.fine("Traversal batch for connector " + connectorName
+ " is completing at the request of the Pusher,"
+ " after processing " + counter + " documents.");
break;
}
} catch (SkippedDocumentException e) {
/* TODO (bmj): This is a temporary solution and should be replaced.
* It uses Exceptions for non-exceptional cases.
*/
// Skip this document. Proceed on to the next one.
logSkippedDocument(docid, e);
} catch (RepositoryDocumentException e) {
// Skip individual documents that fail. Proceed on to the next one.
logSkippedDocument(docid, e);
} catch (RuntimeException e) {
// Skip individual documents that fail. Proceed on to the next one.
logSkippedDocument(docid, e);
}
}
// No more documents. Wrap up any accumulated feed data and send it off.
if (!isCancelled()) {
pusher.flush();
}
} catch (OutOfMemoryError e) {
pusher.cancel();
System.runFinalization();
System.gc();
result = new BatchResult(TraversalDelayPolicy.ERROR);
try {
LOGGER.severe("Out of JVM Heap Space. Will retry later.");
LOGGER.log(Level.FINEST, e.getMessage(), e);
} catch (Throwable t) {
// OutOfMemory state may prevent us from logging the error.
// Don't make matters worse by rethrowing something meaningless.
}
} catch (RepositoryException e) {
// Drop the entire batch on the floor. Do not call checkpoint
// (as there is a discrepancy between what the Connector thinks
// it has fed, and what actually has been pushed).
LOGGER.log(Level.SEVERE, "Repository Exception during traversal.", e);
result = new BatchResult(TraversalDelayPolicy.ERROR);
} catch (PushException e) {
LOGGER.log(Level.SEVERE, "Push Exception during traversal.", e);
// Drop the entire batch on the floor. Do not call checkpoint
// (as there is a discrepancy between what the Connector thinks
// it has fed, and what actually has been pushed).
result = new BatchResult(TraversalDelayPolicy.ERROR);
} catch (FeedException e) {
LOGGER.log(Level.SEVERE, "Feed Exception during traversal.", e);
// Drop the entire batch on the floor. Do not call checkpoint
// (as there is a discrepancy between what the Connector thinks
// it has fed, and what actually has been pushed).
result = new BatchResult(TraversalDelayPolicy.ERROR);
} catch (Throwable t) {
LOGGER.log(Level.SEVERE, "Uncaught Exception during traversal.", t);
// Drop the entire batch on the floor. Do not call checkpoint
// (as there is a discrepancy between what the Connector thinks
// it has fed, and what actually has been pushed).
result = new BatchResult(TraversalDelayPolicy.ERROR);
} finally {
// If we have cancelled the work, abandon the batch.
if (isCancelled()) {
result = new BatchResult(TraversalDelayPolicy.ERROR);
}
// Checkpoint completed work as well as skip past troublesome documents
// (e.g. documents that are too large and will always fail).
if ((result == null) && (checkpointAndSave(resultSet) == null)) {
// Unable to get a checkpoint, so wait a while, then retry batch.
result = new BatchResult(TraversalDelayPolicy.ERROR);
}
}
if (result == null) {
result = new BatchResult(TraversalDelayPolicy.IMMEDIATE, counter,
startTime, clock.getTimeMillis());
} else if (pusher != null) {
// We are returning an error from this batch. Cancel any feed that
// might be in progress.
pusher.cancel();
}
return result;
}