public String importEventSources() {
SimpleDateFormat eventDateFmt = new SimpleDateFormat(Patterns.DATE_LONG);
SimpleDateFormat eventDateFmtR = new SimpleDateFormat(Patterns.DATE_LONG_R);
SimpleDateFormat updateDateFmt = new SimpleDateFormat(Patterns.TIMESTAMP_R);
SimpleDateFormat filterFmt = new SimpleDateFormat("yyyyMMddHHmm");
Config markDateConfig = em.find(Config.class, "evtsrc.read.date");
Date markDate = markDateConfig.getDate();
Date maxUpdateDate = markDate;
StringBuilder log = new StringBuilder();
String url = "http://spdwldr00.appspot.com/event/list?f=" + filterFmt.format(markDate);
Document doc = null;
try {
doc = Jsoup.connect(url).get();
} catch (IOException e) {
log.append("[ERROR]: unable to connect to: ").append(url).append(": ")
.append(e.getMessage()).append("\n");
return log.toString();
}
int persisted = 0;
int merged = 0;
int skipped = 0;
int errors = 0;
BaseDao<IntEvent> dao = daoResolver.resolve(IntEvent.class);
Elements trs = doc.select("tbody tr");
for (int i = 0; i < 1; i++) {
Element tr = trs.get(i);
Elements tds = tr.select("td");
String eventUri = tds.get(6).text();
if (StringUtils.isNotBlank(eventUri)) {
eventUri = eventUri.trim();
} else {
log.append("[ERROR]: event uri is blank").append("\n");
errors++;
continue;
}
// create new task to be persisted/merged
Date operDate = new Date();
IntEvent intEvent = new IntEvent();
intEvent.setStatus(IntEvent.Status.REGISTERED);
intEvent.setEventUri(eventUri);
intEvent.getFingerprint().setInsertDate(operDate);
intEvent.getFingerprint().setUpdateDate(operDate);
intEvent.setEventGroup(tds.get(2).text());
intEvent.setEventName(tds.get(3).text());
String dateString = tds.get(4).text() + " " + tds.get(5).text();
try {
intEvent.setEventDate(eventDateFmt.parse(dateString));
} catch (ParseException e) {
log.append("[ERROR]: unable to parse date string: ").append(dateString)
.append(": ").append(e.getMessage()).append("\n");
errors++;
continue;
}
String updateDateString = tds.get(1).text();
try {
Date updateDate = updateDateFmt.parse(updateDateString);
if (updateDate.compareTo(maxUpdateDate) > 0) {
maxUpdateDate = updateDate;
}
} catch (ParseException e) {
log.append("[WARN]: unable to parse update date string: ").append(dateString)
.append(": ").append(e.getMessage()).append("\n");
}
// find persistent task with matching event uri
EventTaskQuery query = new EventTaskQuery();
query.getEntity().setEventUri(eventUri);
IntEvent persistent = dao.findUnique(query);
StringBuilder detailLog = null;
try {
boolean changed = false;
if (persistent == null) {
dao.persist(intEvent);
log.append("[INFO]: persisted: ");
changed = true;
persisted++;
} else {
CompareToBuilder cmp = new CompareToBuilder();
cmp.append(intEvent.getEventDate(), persistent.getEventDate());
cmp.append(intEvent.getEventGroup(), persistent.getEventGroup());
cmp.append(intEvent.getEventName(), persistent.getEventName());
if (cmp.toComparison() != 0) {
Date eventDate = intEvent.getEventDate();
String eventName = intEvent.getEventName();
String eventGroup = intEvent.getEventGroup();
detailLog = new StringBuilder();
Utils.append(detailLog, "|",
DateUtils.format(persistent.getEventDate(), eventDateFmtR, "null"),
" --> ", DateUtils.format(eventDate, eventDateFmtR, "null"));
Utils.append(detailLog, "|", persistent.getEventGroup(), " --> ",
eventGroup);
Utils.append(detailLog, "|", persistent.getEventName(), " --> ", eventName);
persistent.setEventDate(eventDate);
persistent.setEventGroup(eventGroup);
persistent.setEventName(eventName);
intEvent = dao.merge(persistent);
log.append("[INFO]: merged: ");
changed = true;
merged++;
} else {
skipped++;
}
}
if (changed) {
log.append(eventDateFmtR.format(intEvent.getEventDate())).append(", ")
.append(intEvent.getEventGroup()).append(", ")
.append(intEvent.getEventName())
.append(detailLog == null ? "" : detailLog.toString()).append("\n");
}
} catch (Exception e) {
detailLog = null;
log.append("[ERROR]: unable to persist/merge").append("\n");
errors++;
continue;
}
}
log.append(String.format("Processed: persisted: %d, merged: %d, skipped: %d, errors: %d\n",
persisted, merged, skipped, errors));
markDateConfig.setDate(maxUpdateDate);
markDateConfig = em.merge(markDateConfig);
log.append("Updated ").append(markDateConfig.getCode()).append(" to ")
.append(updateDateFmt.format(markDateConfig.getDate()));
em.flush();
throw new RuntimeException("ERROR");
// return log.toString();
}