if (closed) {
return;
}
// Let's call the Rss flow
SyndFeed feed = getFeed(url);
if (feed != null) {
if (logger.isDebugEnabled()) logger.debug("Reading feed from {}", url);
Date feedDate = feed.getPublishedDate();
if (logger.isDebugEnabled()) logger.debug("Feed publish date is {}", feedDate);
String lastupdateField = "_lastupdated_" + UUID.nameUUIDFromBytes(url.getBytes()).toString();
Date lastDate = getLastDateFromRiver(lastupdateField);
// Comparing dates to see if we have something to do or not
if (lastDate == null || (feedDate != null && feedDate.after(lastDate))) {
// We have to send results to ES
if (logger.isTraceEnabled()) logger.trace("Feed is updated : {}", feed);
try {
// We have now to send each feed to ES
Date mostRecentItemDate = null;
for (SyndEntry message : (Iterable<SyndEntry>) feed.getEntries()) {
// We don't have a global date, so let's see if we have one in items
if (feedDate == null) {
if (message.getUpdatedDate() != null) {
if (lastDate == null || message.getUpdatedDate().after(lastDate)) {
if (mostRecentItemDate == null || message.getUpdatedDate().after(mostRecentItemDate)) {
mostRecentItemDate = message.getUpdatedDate();
if (logger.isTraceEnabled()) logger.trace("No feed date. Using item updated date : {}", feedDate);
}
}
}
if (message.getPublishedDate() != null) {
if (lastDate == null || message.getPublishedDate().after(lastDate)) {
if (mostRecentItemDate == null || message.getPublishedDate().after(mostRecentItemDate)) {
mostRecentItemDate = message.getPublishedDate();
if (logger.isTraceEnabled()) logger.trace("No feed date. Using item published date : {}", feedDate);
}
}
}
}
String description = "";
if (message.getDescription() != null) {
description = message.getDescription().getValue();
}
// Let's define the rule for UUID generation
String id = UUID.nameUUIDFromBytes(description.getBytes()).toString();
// Let's look if object already exists
GetResponse oldMessage = client.prepareGet(indexName, typeName, id).execute().actionGet();
if (!oldMessage.isExists()) {
bulkProcessor.add(indexRequest(indexName).type(typeName).id(id).source(toJson(message, riverName.getName(), feedname, raw)));
if (logger.isDebugEnabled()) logger.debug("FeedMessage update detected for source [{}]", feedname != null ? feedname : "undefined");
if (logger.isTraceEnabled()) logger.trace("FeedMessage is : {}", message);
} else {
if (logger.isTraceEnabled()) logger.trace("FeedMessage {} already exist. Ignoring", id);
}
}
if (feedDate == null) {
feedDate = mostRecentItemDate;
}
if (logger.isTraceEnabled()) {
logger.trace("processing [_seq ]: [{}]/[{}]/[{}], last_seq [{}]", indexName, riverName.name(), lastupdateField, feedDate);
}
// We store the lastupdate date
bulkProcessor.add(indexRequest("_river").type(riverName.name()).id(lastupdateField)
.source(jsonBuilder().startObject().startObject("rss").field(lastupdateField, feedDate).endObject().endObject()));
} catch (IOException e) {
logger.warn("failed to add feed message entry to bulk indexing");
}
} else {
// Nothing new... Just relax !
if (logger.isDebugEnabled()) logger.debug("Nothing new in the feed... Relaxing...");
}
// #8 : Use the ttl rss field to auto adjust feed refresh rate
if (!ignoreTtl && feed.originalWireFeed() != null && feed.originalWireFeed() instanceof Channel) {
Channel channel = (Channel) feed.originalWireFeed();
if (channel.getTtl() > 0) {
int minutes = channel.getTtl();
if (minutes != updateRate.minutes()) {
updateRate = TimeValue.timeValueMinutes(minutes);
if (logger.isInfoEnabled())