throw new RuntimeException(e);
}
if (tabletsInRange == null) {
log.info("Reporting tablet " + extent + " assignment failure: unable to verify Tablet Information");
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extent));
synchronized (openingTablets) {
openingTablets.remove(extent);
openingTablets.notifyAll();
}
return;
}
// If extent given is not the one to be opened, update
if (tabletsInRange.size() != 1 || !tabletsInRange.containsKey(extent)) {
tabletsKeyValues.clear();
synchronized (openingTablets) {
openingTablets.remove(extent);
openingTablets.notifyAll();
for (KeyExtent e : tabletsInRange.keySet())
openingTablets.add(e);
}
} else {
// remove any metadata entries for the previous tablet
Iterator<Key> iter = tabletsKeyValues.keySet().iterator();
Text row = extent.getMetadataEntry();
while (iter.hasNext()) {
Key key = iter.next();
if (!key.getRow().equals(row)) {
iter.remove();
}
}
}
if (tabletsInRange.size() > 1) {
log.debug("Master didn't know " + extent + " was split, letting it know about " + tabletsInRange.keySet());
enqueueMasterMessage(new SplitReportMessage(extent, tabletsInRange));
}
// create the tablet object
for (Entry<KeyExtent,Text> entry : tabletsInRange.entrySet()) {
Tablet tablet = null;
boolean successful = false;
final KeyExtent extentToOpen = entry.getKey();
Text locationToOpen = entry.getValue();
if (onlineTablets.containsKey(extentToOpen)) {
// know this was from fixing a split, because initial check
// would have caught original extent
log.warn("Something is screwy! Already serving tablet " + extentToOpen + " derived from fixing split. Original extent = " + extent);
synchronized (openingTablets) {
openingTablets.remove(extentToOpen);
openingTablets.notifyAll();
}
continue;
}
try {
TabletResourceManager trm = resourceManager.createTabletResourceManager();
// this opens the tablet file and fills in the endKey in the
// extent
tablet = new Tablet(TabletServer.this, locationToOpen, extentToOpen, trm, tabletsKeyValues);
/*
* If a minor compaction starts after a tablet opens, this indicates a log recovery occurred. This recovered data must be minor compacted.
*
* There are three reasons to wait for this minor compaction to finish before placing the tablet in online tablets.
*
* 1) The log recovery code does not handle data written to the tablet on multiple tablet servers. 2) The log recovery code does not block if memory
* is full. Therefore recovering lots of tablets that use a lot of memory could run out of memory. 3) The minor compaction finish event did not make
* it to the logs (the file will be in !METADATA, preventing replay of compacted data)... but do not want a majc to wipe the file out from !METADATA
* and then have another process failure... this could cause duplicate data to replay
*/
if (tablet.getNumEntriesInMemory() > 0 && !tablet.minorCompactNow()) {
throw new RuntimeException("Minor compaction after recovery fails for " + extentToOpen);
}
Assignment assignment = new Assignment(extentToOpen, getTabletSession());
TabletStateStore.setLocation(assignment);
synchronized (openingTablets) {
synchronized (onlineTablets) {
openingTablets.remove(extentToOpen);
onlineTablets.put(extentToOpen, tablet);
openingTablets.notifyAll();
recentlyUnloadedCache.remove(tablet.getExtent());
}
}
tablet = null; // release this reference
successful = true;
} catch (Throwable e) {
log.warn("exception trying to assign tablet " + extentToOpen + " " + locationToOpen, e);
if (e.getMessage() != null)
log.warn(e.getMessage());
String table = extent.getTableId().toString();
ProblemReports.getInstance().report(new ProblemReport(table, TABLET_LOAD, extentToOpen.getUUID().toString(), getClientAddressString(), e));
}
if (!successful) {
synchronized (unopenedTablets) {
synchronized (openingTablets) {
openingTablets.remove(extentToOpen);
unopenedTablets.add(extentToOpen);
openingTablets.notifyAll();
}
}
log.warn("failed to open tablet " + extentToOpen + " reporting failure to master");
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extentToOpen));
long reschedule = Math.min((1l << Math.min(32, retryAttempt)) * 1000, 10 * 60 * 1000l);
log.warn(String.format("rescheduling tablet load in %.2f seconds", reschedule / 1000.));
SimpleTimer.getInstance().schedule(new TimerTask() {
@Override
public void run() {
log.info("adding tablet " + extent + " back to the assignment pool (retry " + retryAttempt + ")");
AssignmentHandler handler = new AssignmentHandler(extentToOpen, retryAttempt + 1);
if (isMetaDataTablet) {
if (Constants.ROOT_TABLET_EXTENT.equals(extent)) {
new Thread(new LoggingRunnable(log, handler), "Root tablet assignment retry").start();
} else {
resourceManager.addMetaDataAssignment(handler);
}
} else {
resourceManager.addAssignment(handler);
}
}
}, reschedule);
} else {
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOADED, extentToOpen));
}
}
}