private void recoverIndex(Transaction tx) throws IOException {
long start = System.currentTimeMillis();
// It is possible index updates got applied before the journal updates..
// in that case we need to removed references to Jobs that are not in the journal
final Location lastAppendLocation = journal.getLastAppendLocation();
long undoCounter = 0;
// Go through all the jobs in each scheduler and check if any are added after
// the last appended location and remove those. For now we ignore the update
// location since the scheduled job will update itself after the next fire and
// a new update will replace any existing update.
for (Iterator<Map.Entry<String, JobSchedulerImpl>> i = metaData.getJobSchedulers().iterator(tx); i.hasNext();) {
Map.Entry<String, JobSchedulerImpl> entry = i.next();
JobSchedulerImpl scheduler = entry.getValue();
List<JobLocation> jobs = scheduler.getAllScheduledJobs(tx);
for (JobLocation job : jobs) {
if (job.getLocation().compareTo(lastAppendLocation) >= 0) {
if (scheduler.removeJobAtTime(tx, job.getJobId(), job.getNextTime())) {
LOG.trace("Removed Job past last appened in the journal: {}", job.getJobId());
undoCounter++;
}
}
}
}
if (undoCounter > 0) {
// The rolled back operations are basically in flight journal writes. To avoid getting
// these the end user should do sync writes to the journal.
long end = System.currentTimeMillis();
LOG.info("Rolled back {} messages from the index in {} seconds.", undoCounter, ((end - start) / 1000.0f));
undoCounter = 0;
}
// Now we check for missing and corrupt journal files.
// 1. Collect the set of all referenced journal files based on the Location of the
// the scheduled jobs and the marked last update field.
HashSet<Integer> missingJournalFiles = new HashSet<Integer>();
for (Iterator<Map.Entry<String, JobSchedulerImpl>> i = metaData.getJobSchedulers().iterator(tx); i.hasNext();) {
Map.Entry<String, JobSchedulerImpl> entry = i.next();
JobSchedulerImpl scheduler = entry.getValue();
List<JobLocation> jobs = scheduler.getAllScheduledJobs(tx);
for (JobLocation job : jobs) {
missingJournalFiles.add(job.getLocation().getDataFileId());
if (job.getLastUpdate() != null) {
missingJournalFiles.add(job.getLastUpdate().getDataFileId());
}
}
}
// 2. Remove from that set all known data file Id's in the journal and what's left
// is the missing set which will soon also contain the corrupted set.
missingJournalFiles.removeAll(journal.getFileMap().keySet());
if (!missingJournalFiles.isEmpty()) {
LOG.info("Some journal files are missing: {}", missingJournalFiles);
}
// 3. Now check all references in the journal logs for corruption and add any
// corrupt journal files to the missing set.
HashSet<Location> corruptedLocations = new HashSet<Location>();
if (isCheckForCorruptJournalFiles()) {
Collection<DataFile> dataFiles = journal.getFileMap().values();
for (DataFile dataFile : dataFiles) {
int id = dataFile.getDataFileId();
for (long offset : dataFile.getCorruptedBlocks()) {
corruptedLocations.add(new Location(id, (int) offset));
}
}
if (!corruptedLocations.isEmpty()) {
LOG.debug("Found some corrupted data blocks in the journal: {}", corruptedLocations.size());