/*-
* See the file LICENSE for redistribution information.
*
* Copyright (c) 2002-2005
* Sleepycat Software. All rights reserved.
*
* $Id: Cleaner.java,v 1.158 2005/09/21 15:47:14 cwl Exp $
*/
package com.sleepycat.je.cleaner;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.EnvironmentStats;
import com.sleepycat.je.StatsConfig;
import com.sleepycat.je.config.EnvironmentParams;
import com.sleepycat.je.dbi.DatabaseId;
import com.sleepycat.je.dbi.DatabaseImpl;
import com.sleepycat.je.dbi.DbConfigManager;
import com.sleepycat.je.dbi.DbTree;
import com.sleepycat.je.dbi.EnvironmentImpl;
import com.sleepycat.je.dbi.MemoryBudget;
import com.sleepycat.je.latch.Latch;
import com.sleepycat.je.log.CleanerFileReader;
import com.sleepycat.je.log.FileManager;
import com.sleepycat.je.tree.BIN;
import com.sleepycat.je.tree.ChildReference;
import com.sleepycat.je.tree.DIN;
import com.sleepycat.je.tree.IN;
import com.sleepycat.je.tree.LN;
import com.sleepycat.je.tree.Node;
import com.sleepycat.je.tree.SearchResult;
import com.sleepycat.je.tree.Tree;
import com.sleepycat.je.tree.TreeLocation;
import com.sleepycat.je.tree.WithRootLatched;
import com.sleepycat.je.txn.BasicLocker;
import com.sleepycat.je.txn.LockGrantType;
import com.sleepycat.je.utilint.DaemonThread;
import com.sleepycat.je.utilint.DbLsn;
import com.sleepycat.je.utilint.PropUtil;
import com.sleepycat.je.utilint.Tracer;
/**
* The Cleaner is responsible for effectively garbage collecting the JE log.
* It looks through log files and locates log records (IN's and LN's of all
* flavors) that are superceded by later versions. Those that are "current"
* are propagated to a newer log file so that older log files can be deleted.
*/
public class Cleaner extends DaemonThread {
/* From cleaner */
private static final String CLEAN_IN = "CleanIN:";
private static final String CLEAN_LN = "CleanLN:";
private static final String CLEAN_MIGRATE_LN = "CleanMigrateLN:";
private static final String CLEAN_PENDING_LN = "CleanPendingLN:";
private static final boolean DEBUG_TRACING = false;
private EnvironmentImpl env;
private long lockTimeout;
private int readBufferSize;
/* Cumulative counters. */
private int nCleanerRuns = 0;
private int nCleanerDeletions = 0;
private int nINsObsolete = 0;
private int nINsCleaned = 0;
private int nINsDead = 0;
private int nINsMigrated = 0;
private int nLNsObsolete = 0;
private int nLNsCleaned = 0;
private int nLNsDead = 0;
private int nLNsLocked = 0;
private int nLNsMigrated = 0;
private int nLNsMarked = 0;
private int nPendingLNsProcessed = 0;
private int nMarkedLNsProcessed = 0;
private int nToBeCleanedLNsProcessed = 0;
private int nClusterLNsProcessed = 0;
private int nPendingLNsLocked = 0;
private int nEntriesRead = 0;
private long nRepeatIteratorReads = 0;
/* Per Run counters. Reset before each invocation of the cleaner. */
private int nINsObsoleteThisRun = 0;
private int nINsCleanedThisRun = 0;
private int nINsDeadThisRun = 0;
private int nINsMigratedThisRun = 0;
private int nLNsObsoleteThisRun = 0;
private int nLNsCleanedThisRun = 0;
private int nLNsDeadThisRun = 0;
private int nLNsLockedThisRun = 0;
private int nLNsMigratedThisRun = 0;
private int nLNsMarkedThisRun = 0;
private int nEntriesReadThisRun;
private long nRepeatIteratorReadsThisRun;
private boolean expunge;
private boolean clusterResident;
private boolean clusterAll;
private int maxBatchFiles;
private Long currentFile;
private List toBeCleanedFiles;
private FileSelector fileSelector;
private UtilizationProfile profile;
private Set lowUtilizationFiles;
private Level detailedTraceLevel; // level value for detailed trace msgs
private Object deleteFileLock;
public Cleaner(EnvironmentImpl env, long waitTime, String name)
throws DatabaseException {
super(waitTime, name, env);
this.env = env;
profile = env.getUtilizationProfile();
DbConfigManager cm = env.getConfigManager();
lockTimeout = PropUtil.microsToMillis(cm.getLong
(EnvironmentParams.CLEANER_LOCK_TIMEOUT));
readBufferSize = cm.getInt(EnvironmentParams.CLEANER_READ_SIZE);
if (readBufferSize <= 0) {
readBufferSize = cm.getInt
(EnvironmentParams.LOG_ITERATOR_READ_SIZE);
}
expunge = cm.getBoolean(EnvironmentParams.CLEANER_REMOVE);
clusterResident = cm.getBoolean(EnvironmentParams.CLEANER_CLUSTER);
clusterAll = cm.getBoolean(EnvironmentParams.CLEANER_CLUSTER_ALL);
maxBatchFiles = cm.getInt(EnvironmentParams.CLEANER_MAX_BATCH_FILES);
if (clusterResident && clusterAll) {
throw new IllegalArgumentException
("Both " + EnvironmentParams.CLEANER_CLUSTER +
" and " + EnvironmentParams.CLEANER_CLUSTER_ALL +
" may not be set to true.");
}
fileSelector = new FileSelector();
toBeCleanedFiles = Collections.EMPTY_LIST;
lowUtilizationFiles = Collections.EMPTY_SET;
deleteFileLock = new Object();
detailedTraceLevel = Tracer.parseLevel
(env, EnvironmentParams.JE_LOGGING_LEVEL_CLEANER);
}
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("<Cleaner name=\"").append(name).append("\"/>");
return sb.toString();
}
/**
* Cleaner doesn't have a work queue so just throw an exception if it's
* ever called.
*/
public void addToQueue(Object o)
throws DatabaseException {
throw new DatabaseException
("Cleaner.addToQueue should never be called.");
}
/**
* Load stats.
*/
public void loadStats(StatsConfig config, EnvironmentStats stat)
throws DatabaseException {
stat.setCleanerBacklog(toBeCleanedFiles.size());
stat.setNCleanerRuns(nCleanerRuns);
stat.setNCleanerDeletions(nCleanerDeletions);
stat.setNINsObsolete(nINsObsolete);
stat.setNINsCleaned(nINsCleaned);
stat.setNINsDead(nINsDead);
stat.setNINsMigrated(nINsMigrated);
stat.setNLNsObsolete(nLNsObsolete);
stat.setNLNsCleaned(nLNsCleaned);
stat.setNLNsDead(nLNsDead);
stat.setNLNsLocked(nLNsLocked);
stat.setNLNsMigrated(nLNsMigrated);
stat.setNLNsMarked(nLNsMarked);
stat.setNPendingLNsProcessed(nPendingLNsProcessed);
stat.setNMarkedLNsProcessed(nMarkedLNsProcessed);
stat.setNToBeCleanedLNsProcessed(nToBeCleanedLNsProcessed);
stat.setNClusterLNsProcessed(nClusterLNsProcessed);
stat.setNPendingLNsLocked(nPendingLNsLocked);
stat.setNCleanerEntriesRead(nEntriesRead);
stat.setNRepeatIteratorReads(nRepeatIteratorReads);
if (config.getClear()) {
nCleanerRuns = 0;
nCleanerDeletions = 0;
nINsObsolete = 0;
nINsCleaned = 0;
nINsDead = 0;
nINsMigrated = 0;
nLNsObsolete = 0;
nLNsCleaned = 0;
nLNsDead = 0;
nLNsLocked = 0;
nLNsMigrated = 0;
nLNsMarked = 0;
nPendingLNsProcessed = 0;
nMarkedLNsProcessed = 0;
nToBeCleanedLNsProcessed = 0;
nClusterLNsProcessed = 0;
nPendingLNsLocked = 0;
nEntriesRead = 0;
nRepeatIteratorReads = 0;
}
}
public synchronized void clearEnv() {
env = null;
}
/**
* Return the number of retries when a deadlock exception occurs.
*/
protected int nDeadlockRetries()
throws DatabaseException {
return env.getConfigManager().getInt
(EnvironmentParams.CLEANER_DEADLOCK_RETRY);
}
/**
* Called whenever the daemon thread wakes up from a sleep.
*/
public void onWakeup()
throws DatabaseException {
doClean(true, // invokedFromDaemon
true, // cleanMultipleFiles
false); // forceCleaning
}
/**
* Cleans selected files and returns the number of files cleaned. May be
* called by the daemon thread or programatically.
*
* @param invokedFromDaemon currently has no effect.
*
* @param cleanMultipleFiles is true to clean until we're under budget,
* or false to clean at most one file.
*
* @param forceCleaning is true to clean even if we're not under the
* utilization threshold.
*
* @return the number of files deleted, not including files cleaned
* unsuccessfully.
*/
public synchronized int doClean(boolean invokedFromDaemon,
boolean cleanMultipleFiles,
boolean forceCleaning)
throws DatabaseException {
if (env.isClosed()) {
return 0;
}
/* Clean until no more files are selected. */
int nOriginalLogFiles = profile.getNumberOfFiles();
int nFilesCleaned = 0;
while (true) {
/* Don't clean forever. */
if (nFilesCleaned >= nOriginalLogFiles) {
break;
}
/* Stop if the daemon is shut down. */
if (isShutdownRequested()) {
break;
}
/*
* Process pending LNs and then attempt to delete all cleaned files
* that are safe to delete. Pending LNs can prevent file deletion.
*/
processPending();
deleteSafeToDeleteFiles();
/*
* Select a file for cleaning and get a new low utilization file
* set. Update the lowUtilizationFiles and toBeCleanedFiles fields
* with the new sets atomically, since they are used by other
* threads. These sets are read-only after assignment, so no
* synchronization is needed.
*/
Set newLowUtilizationSet = (clusterResident || clusterAll) ?
(new HashSet()) : null;
toBeCleanedFiles = fileSelector.selectFilesForCleaning
(profile, forceCleaning, newLowUtilizationSet, maxBatchFiles);
if (newLowUtilizationSet != null) {
lowUtilizationFiles = newLowUtilizationSet;
}
/*
* If no file was selected, the total utilization is under the
* threshold and we can stop cleaning.
*/
if (toBeCleanedFiles.size() == 0) {
break;
}
/*
* Clean the selected file.
*/
resetPerRunCounters();
boolean finished = false;
currentFile = profile.getCheapestFileToClean(toBeCleanedFiles);
long fileNumValue = currentFile.longValue();
try {
nCleanerRuns++;
assert Latch.countLatchesHeld() == 0;
String traceMsg =
"CleanerRun " + nCleanerRuns +
" on file 0x" + Long.toHexString(fileNumValue) +
" begins backlog=" + toBeCleanedFiles.size();
Tracer.trace(Level.INFO, env, traceMsg);
if (DEBUG_TRACING) {
System.out.println("\n" + traceMsg);
}
/* Clean all log entries in the file. */
processFile(currentFile);
/* Update status information. */
fileSelector.addCleanedFile(currentFile);
nFilesCleaned += 1;
accumulatePerRunCounters();
finished = true;
} catch (IOException IOE) {
Tracer.trace(env, "Cleaner", "doClean", "", IOE);
throw new DatabaseException(IOE);
} finally {
currentFile = null;
String traceMsg =
"CleanerRun " + nCleanerRuns +
" on file 0x" + Long.toHexString(fileNumValue) +
" invokedFromDaemon=" + invokedFromDaemon +
" finished=" + finished +
" nEntriesRead=" + nEntriesReadThisRun +
" nINsObsolete=" + nINsObsoleteThisRun +
" nINsCleaned=" + nINsCleanedThisRun +
" nINsDead=" + nINsDeadThisRun +
" nINsMigrated=" + nINsMigratedThisRun +
" nLNsObsolete=" + nLNsObsoleteThisRun +
" nLNsCleaned=" + nLNsCleanedThisRun +
" nLNsDead=" + nLNsDeadThisRun +
" nLNsMigrated=" + nLNsMigratedThisRun +
" nLNsMarked=" + nLNsMarkedThisRun +
" nLNsLocked=" + nLNsLockedThisRun;
Tracer.trace(Level.SEVERE, env, traceMsg);
if (DEBUG_TRACING) {
System.out.println("\n" + traceMsg);
}
}
/* If we should only clean one file, stop now. */
if (!cleanMultipleFiles) {
break;
}
}
return nFilesCleaned;
}
/**
* Deletes all files that are safe-to-delete, if an exclusive lock on the
* environment can be obtained.
*/
private void deleteSafeToDeleteFiles()
throws DatabaseException {
/*
* Synchronized to prevent multiple threads from requesting the same
* file lock.
*/
synchronized (deleteFileLock) {
Set safeFiles = fileSelector.copySafeToDeleteFiles();
if (safeFiles == null) {
return; /* Nothing to do. */
}
/*
* Fail loudly if the environment is invalid. A
* RunRecoveryException must have occurred.
*/
env.checkIfInvalid();
/*
* Fail silent if the environment is not open.
*/
if (env.mayNotWrite()) {
return;
}
/*
* If we can't get an exclusive lock, then there are reader
* processes and we can't delete any cleaned files.
*/
if (!env.getFileManager().lockEnvironment(false, true)) {
Tracer.trace
(Level.SEVERE, env, "Cleaner has " + safeFiles.size() +
" files not deleted because of read-only processes.");
return;
}
try {
for (Iterator i = safeFiles.iterator(); i.hasNext();) {
Long fileNum = (Long) i.next();
long fileNumValue = fileNum.longValue();
boolean deleted = false;
try {
if (expunge) {
env.getFileManager().deleteFile(fileNumValue);
} else {
env.getFileManager().renameFile
(fileNumValue, FileManager.DEL_SUFFIX);
}
deleted = true;
} catch (DatabaseException e) {
traceFileNotDeleted(e, fileNumValue);
} catch (IOException e) {
traceFileNotDeleted(e, fileNumValue);
}
if (deleted) {
profile.removeFile(fileNum);
fileSelector.removeDeletedFile(fileNum);
Tracer.trace
(Level.SEVERE, env,
"Cleaner deleted file 0x" +
Long.toHexString(fileNumValue));
}
nCleanerDeletions++;
}
} finally {
env.getFileManager().releaseExclusiveLock();
}
}
}
private void traceFileNotDeleted(Exception e, long fileNum) {
Tracer.trace
(env, "Cleaner", "deleteSafeToDeleteFiles",
"Log file 0x" + Long.toHexString(fileNum) + " could not be " +
(expunge ? "deleted" : "renamed") +
". This operation will be retried at the next checkpoint.",
e);
}
/**
* Returns a copy of the cleaned and processed files at the time a
* checkpoint starts.
*
* <p>If non-null is returned, the checkpoint should flush an extra level,
* and addCheckpointedFiles() should be called when the checkpoint is
* complete.</p>
*/
public Set[] getFilesAtCheckpointStart()
throws DatabaseException {
/* Pending LNs can prevent file deletion. */
processPending();
return fileSelector.getFilesAtCheckpointStart();
}
/**
* When a checkpoint is complete, update the files that were returned at
* the beginning of the checkpoint.
*/
public void updateFilesAtCheckpointEnd(Set[] files)
throws DatabaseException {
fileSelector.updateFilesAtCheckpointEnd(files);
deleteSafeToDeleteFiles();
}
/**
* Process all log entries in the given file.
*/
private void processFile(Long fileNum)
throws DatabaseException, IOException {
/* Get the current obsolete offsets for this file. */
PackedOffsets obsoleteOffsets = new PackedOffsets();
TrackedFileSummary tfs =
profile.getObsoleteDetail(fileNum, obsoleteOffsets);
PackedOffsets.Iterator obsoleteIter = obsoleteOffsets.iterator();
long nextObsolete = -1;
/*
* Add the overhead of this method to the budget. The log size of the
* offsets happens to be the same as the memory overhead.
*/
MemoryBudget budget = env.getMemoryBudget();
int adjustMem = readBufferSize + obsoleteOffsets.getLogSize();
budget.updateMiscMemoryUsage(adjustMem);
try {
/* Create the file reader. */
CleanerFileReader reader = new CleanerFileReader
(env, readBufferSize, DbLsn.NULL_LSN, fileNum);
DbTree dbMapTree = env.getDbMapTree();
TreeLocation location = new TreeLocation();
while (reader.readNextEntry()) {
nEntriesRead += 1;
long lsn = reader.getLastLsn();
long fileOffset = DbLsn.getFileOffset(lsn);
boolean isObsolete = false;
/* Check for a known obsolete node. */
while (nextObsolete < fileOffset && obsoleteIter.hasNext()) {
nextObsolete = obsoleteIter.next();
}
if (nextObsolete == fileOffset) {
isObsolete = true;
}
/* Check for a deleted LN next because it is very cheap. */
if (!isObsolete &&
reader.isLN() &&
reader.getLN().isDeleted()) {
/* Deleted LNs are always obsolete. */
isObsolete = true;
}
/* Check current tracker last, as it is more expensive. */
if (!isObsolete &&
tfs != null &&
tfs.containsObsoleteOffset(fileOffset)) {
isObsolete = true;
}
/* Skip known obsolete nodes. */
if (isObsolete) {
if (reader.isLN()) {
nLNsObsoleteThisRun++;
} else if (reader.isIN()) {
nINsObsoleteThisRun++;
}
continue;
}
/* Evict before processing each entry. */
env.getEvictor().doCriticalEviction();
/* The entry is not known to be obsolete -- process it now. */
if (reader.isLN()) {
LN targetLN = reader.getLN();
DatabaseId dbId = reader.getDatabaseId();
DatabaseImpl db = dbMapTree.getDb(dbId, lockTimeout);
processLN
(targetLN, db, reader.getKey(),
reader.getDupTreeKey(), lsn, location);
} else if (reader.isIN()) {
IN targetIN = reader.getIN();
DatabaseId dbId = reader.getDatabaseId();
DatabaseImpl db = dbMapTree.getDb(dbId, lockTimeout);
targetIN.setDatabase(db);
processIN(targetIN, db, lsn);
} else if (reader.isRoot()) {
env.rewriteMapTreeRoot(lsn);
}
/*
* Process pending LNs before proceeding in order to prevent
* the pending list from growing too large.
*/
processPending();
}
nEntriesReadThisRun = reader.getNumRead();
nRepeatIteratorReadsThisRun = reader.getNRepeatIteratorReads();
} finally {
/* Subtract the overhead of this method to the budget. */
budget.updateMiscMemoryUsage(0 - adjustMem);
/* Allow flushing of TFS when cleaning is complete. */
if (tfs != null) {
tfs.setAllowFlush(true);
}
}
}
/**
* Processes an LN after looking up its parent BIN.
*/
private void processLN(LN ln,
DatabaseImpl db,
byte[] key,
byte[] dupKey,
long logLsn,
TreeLocation location)
throws DatabaseException {
/* Status variables are used to generate debug tracing info. */
boolean obsolete = false; // The LN is no longer in use.
boolean migrated = false; // The LN was in use and is migrated.
boolean lockDenied = false;// The LN lock was denied.
boolean completed = false; // This method completed.
long nodeId = ln.getNodeId();
BasicLocker locker = null;
BIN bin = null;
DIN parentDIN = null; // for DupCountLNs
try {
nLNsCleanedThisRun++;
/* The whole database is gone, so this LN is obsolete. */
if (db == null || db.getIsDeleted()) {
nLNsDeadThisRun++;
completed = true;
return;
}
Tree tree = db.getTree();
assert tree != null;
/*
* Search down to the bottom most level for the parent of this LN.
*/
boolean parentFound = tree.getParentBINForChildLN
(location, key, dupKey, ln,
false, // splitsAllowed
true, // findDeletedEntries
false, // searchDupTree
false); // updateGeneration
bin = location.bin;
int index = location.index;
if (!parentFound) {
nLNsDeadThisRun++;
completed = true;
return;
}
/*
* Now we're at the parent for this LN, whether BIN, DBIN or DIN.
* If knownDeleted, LN is deleted and can be purged.
*/
if (bin.isEntryKnownDeleted(index)) {
nLNsDeadThisRun++;
obsolete = true;
completed = true;
return;
}
/*
* Determine whether the parent is the current BIN, or in the case
* of a DupCountLN, a DIN. Get the tree LSN in either case.
*/
boolean lnIsDupCountLN = ln.containsDuplicates();
long treeLsn;
if (lnIsDupCountLN) {
parentDIN = (DIN) bin.fetchTarget(index);
parentDIN.latch(false);
ChildReference dclRef = parentDIN.getDupCountLNRef();
treeLsn = dclRef.getLsn();
} else {
treeLsn = bin.getLsn(index);
}
/*
* Check to see whether the LN being migrated is locked elsewhere.
* Do that by attempting to lock it. We can hold the latch on the
* BIN (and DIN) since we always attempt to acquire a non-blocking
* read lock. Holding the latch ensures that the INs won't change
* underneath us because of splits or eviction.
*/
locker = new BasicLocker(env);
LockGrantType lock = locker.nonBlockingReadLock(nodeId, db);
if (lock == LockGrantType.DENIED) {
/*
* LN is currently locked by another Locker, so we can't assume
* anything about the value of the LSN in the bin. However,
* we can check whether the lock owner's abort LSN is greater
* than the log LSN; if so, the log LSN is obsolete. Before
* doing this we must release all latches to avoid a deadlock.
*/
if (parentDIN != null) {
parentDIN.releaseLatch();
parentDIN = null;
}
bin.releaseLatch();
long abortLsn = locker.getOwnerAbortLsn(nodeId);
if (abortLsn != DbLsn.NULL_LSN &&
DbLsn.compareTo(abortLsn, logLsn) > 0) {
nLNsDeadThisRun++;
obsolete = true;
} else {
nLNsLockedThisRun++;
lockDenied = true;
}
completed = true;
return;
}
/*
* We were able to lock this LN in the tree. Try to migrate this
* LN to the end of the log file so we can throw away the old log
* entry.
*
* 1. If the LSN in the tree and in the log are the same,
* we can migrate it, or discard it if the LN is deleted.
*
* 2. If the LSN in the tree is < the LSN in the log, the
* log entry is obsolete, because this LN has been rolled
* back to a previous version by a txn that aborted.
*
* 3. If the LSN in the tree is > the LSN in the log, the
* log entry is obsolete, because the LN was advanced
* forward by some now-committed txn.
*/
if (treeLsn == logLsn) {
if (ln.isDeleted()) {
/*
* If the LN is deleted, we must set knownDeleted to
* prevent fetching it later. This could occur when
* scanning over deleted entries that have not been
* compressed away [10553].
*/
assert !lnIsDupCountLN;
bin.setKnownDeletedLeaveTarget(index);
nLNsDeadThisRun++;
obsolete = true;
} else {
if (lnIsDupCountLN) {
/*
* Migrate the DupCountLN now to avoid having to
* process the migrate flag for DupCountLNs in all
* other places.
*/
long newLNLsn = ln.log
(env, db.getId(), key, logLsn, locker);
parentDIN.updateDupCountLNRef(newLNLsn);
nLNsMigratedThisRun++;
} else {
/*
* Set the migrate flag and dirty the BIN. The evictor
* or checkpointer will migrate the LN later.
*/
bin.setMigrate(index, true);
/*
* Update the generation so that the BIN is not evicted
* immediately. This allows the cleaner to fill in as
* many entries as possible before eviction, as
* to-be-cleaned files are processed.
*/
bin.setGeneration();
/*
* Set the target node so it does not have to be
* fetched when it is migrated. Must call postFetchInit
* to initialize MapLNs that have not been fully
* initialized yet [#13191].
*/
if (bin.getTarget(index) == null) {
ln.postFetchInit(db, logLsn);
bin.updateEntry(index, ln);
}
nLNsMarkedThisRun++;
}
migrated = true;
}
} else {
/* LN is obsolete and can be purged. */
nLNsDeadThisRun++;
obsolete = true;
}
completed = true;
return;
} finally {
if (parentDIN != null) {
parentDIN.releaseLatchIfOwner();
}
if (bin != null) {
bin.releaseLatchIfOwner();
}
if (locker != null) {
locker.operationEnd();
}
if (completed && lockDenied) {
fileSelector.addPendingLN(ln, db.getId(), key, dupKey);
}
trace(detailedTraceLevel, CLEAN_LN, ln, logLsn,
completed, obsolete, migrated);
}
}
/**
* Add a BIN entry to the pending LN set. This is used in the case where
* a split occurs, and we don't want to migrate the LN during the split
* for performance reasons.
*/
public void handleNoMigrationLogging(BIN bin)
throws DatabaseException {
DatabaseImpl db = bin.getDatabase();
/*
* We must fetch the node here in order to obtain enough information to
* look up the entry later. This is not desirable during a split, but
* in fact the node is rarely non-resident. The migrate flag prevents
* LN stripping, so the only time the node is null is after an abort.
*/
for (int index = 0; index < bin.getNEntries(); index++) {
if (bin.getMigrate(index)) {
/*
* fetchTarget should never return null if the MIGRATE flag is
* set, since the LN is not yet cleaned.
*/
LN ln = (LN) bin.fetchTarget(index);
assert ln != null;
byte[] key = getLNMainKey(bin, index);
byte[] dupKey = getLNDupKey(bin, index, ln);
fileSelector.addPendingLN(ln, db.getId(), key, dupKey);
bin.setMigrate(index, false);
}
}
}
/**
* If any LNs are pending, process them. This method should be called
* often enough to prevent the pending LN set from growing too large.
*/
private void processPending()
throws DatabaseException {
LNInfo[] pendingLNs = fileSelector.getPendingLNs();
if (pendingLNs == null) {
return;
}
DbTree dbMapTree = env.getDbMapTree();
TreeLocation location = new TreeLocation();
for (int i = 0; i < pendingLNs.length; i += 1) {
LNInfo info = pendingLNs[i];
DatabaseId dbId = info.getDbId();
DatabaseImpl db = dbMapTree.getDb(dbId, lockTimeout);
byte[] key = info.getKey();
byte[] dupKey = info.getDupKey();
LN ln = info.getLN();
/* Evict before processing each entry. */
env.getEvictor().doCriticalEviction();
processPendingLN
(ln, db, key, dupKey, location);
}
}
/**
* Processes a pending LN, getting the lock first to ensure that the
* overhead of retries is mimimal.
*/
private void processPendingLN(LN ln,
DatabaseImpl db,
byte[] key,
byte[] dupKey,
TreeLocation location)
throws DatabaseException {
boolean parentFound = false; // We found the parent BIN.
boolean processedHere = true; // The LN was cleaned here.
boolean lockDenied = false; // The LN lock was denied.
boolean obsolete = false; // The LN is no longer in use.
boolean completed = false; // This method completed.
BasicLocker locker = null;
BIN bin = null;
try {
nPendingLNsProcessed++;
/* The whole database is gone, so this LN is obsolete. */
if (db == null || db.getIsDeleted()) {
nLNsDead++;
obsolete = true;
completed = true;
return;
}
Tree tree = db.getTree();
assert tree != null;
/* Get a non-blocking lock on the original node ID. */
locker = new BasicLocker(env);
if (locker.nonBlockingReadLock(ln.getNodeId(), db) ==
LockGrantType.DENIED) {
/* Try again later. */
nPendingLNsLocked++;
lockDenied = true;
completed = true;
return;
}
/*
* Search down to the bottom most level for the parent of this LN.
*
* We pass searchDupTree=true to search the dup tree by nodeID if
* necessary. This handles the case where dupKey is null because
* the pending entry was a deleted single-duplicate in a BIN.
*/
parentFound = tree.getParentBINForChildLN
(location, key, dupKey, ln,
false, // splitsAllowed
true, // findDeletedEntries
true, // searchDupTree
false); // updateGeneration
bin = location.bin;
int index = location.index;
if (!parentFound) {
nLNsDead++;
obsolete = true;
completed = true;
return;
}
migrateLN
(db, bin.getLsn(index), bin, index,
true, // wasCleaned
true, // isPending
ln.getNodeId(), // lockedPendingNodeId
CLEAN_PENDING_LN);
processedHere = false;
completed = true;
} catch (DatabaseException DBE) {
DBE.printStackTrace();
Tracer.trace(env, "com.sleepycat.je.cleaner.Cleaner", "processLN",
"Exception thrown: ", DBE);
throw DBE;
} finally {
if (bin != null) {
bin.releaseLatchIfOwner();
}
if (locker != null) {
locker.operationEnd();
}
/*
* If migrateLN was not called above, remove the pending LN and
* perform tracing in this method.
*/
if (processedHere) {
if (completed && !lockDenied) {
fileSelector.removePendingLN(ln.getNodeId());
}
trace(detailedTraceLevel, CLEAN_PENDING_LN, ln, DbLsn.NULL_LSN,
completed, obsolete, false /*migrated*/);
}
}
}
/**
* Returns whether the given BIN entry may be stripped by the evictor.
* True is always returned if the BIN is not dirty. False is returned if
* the BIN is dirty and the entry will be migrated soon.
*/
public boolean isEvictable(BIN bin, int index) {
if (bin.getDirty()) {
if (bin.getMigrate(index)) {
return false;
}
Long fileNum = new Long(DbLsn.getFileNumber(bin.getLsn(index)));
if (toBeCleanedFiles.contains(fileNum)) {
return false;
}
if (clusterResident &&
bin.getTarget(index) != null &&
lowUtilizationFiles.contains(fileNum)) {
return false;
}
if (clusterAll &&
lowUtilizationFiles.contains(fileNum)) {
return false;
}
}
return true;
}
/**
* This method should be called just before logging a BIN. LNs will be
* migrated if the MIGRATE flag is set, or if they are in a file to be
* cleaned, or if the LNs qualify according to the rules for cluster and
* clusterAll.
*
* <p>On return this method guarantees that no MIGRATE flag will be set on
* any child entry. If this method is *not* called before logging a BIN,
* then the handleNoMigrationLogging method must be called.</p>
*
* @param bin is the latched BIN. The latch will not be released by this
* method.
*/
public void migrateLNs(BIN bin)
throws DatabaseException {
DatabaseImpl db = bin.getDatabase();
boolean isBinInDupDb = db.getSortedDuplicates() &&
!bin.containsDuplicates();
for (int index = 0; index < bin.getNEntries(); index += 1) {
long childLsn = bin.getLsn(index);
long fileNum = DbLsn.getFileNumber(childLsn);
boolean doMigration = false;
boolean wasCleaned = false;
/*
* Select a child entry if it should be migrated for cleaning, or
* if it qualifies for clusterResident or clusterAll migration.
*/
if (bin.getMigrate(index)) {
/*
* Always try to migrate if the MIGRATE flag is set, since
* it has been cleaned. If we did not migrate it, we would
* have to add it to pending LN set.
*/
doMigration = true;
wasCleaned = true;
nMarkedLNsProcessed++;
} else if (isShutdownRequested()) {
/*
* Do nothing if the environment is shutting down and the
* MIGRATE flag is not set. Proactive migration during
* shutdown is counterproductive -- it prevents a short final
* checkpoint, and it does not allow more files to be deleted.
*/
} else if (isBinInDupDb) {
/*
* Do nothing if this is a BIN in a duplicate database. We
* must not fetch DINs, since this BIN may be about to be
* evicted. Fetching a DIN would add it as an orphan to the
* INList, plus an IN with non-LN children is not evictable.
*/
} else if (toBeCleanedFiles.contains(new Long(fileNum))) {
/* Migrate because it will be cleaned soon. */
doMigration = true;
nToBeCleanedLNsProcessed++;
} else if ((clusterResident || clusterAll) &&
lowUtilizationFiles.contains(new Long(fileNum)) &&
(clusterAll || bin.getTarget(index) != null)) {
/* Migrate for clustering. */
doMigration = true;
nClusterLNsProcessed++;
}
if (doMigration) {
migrateLN
(db, childLsn, bin, index,
wasCleaned,
false, // isPending
0, // lockedPendingNodeId
CLEAN_MIGRATE_LN);
}
}
}
/**
* Migrate an LN in the given BIN entry, if it is not obsolete. The BIN is
* latched on entry to this method and is left latched when it returns.
*/
private void migrateLN(DatabaseImpl db,
long lsn,
BIN bin,
int index,
boolean wasCleaned,
boolean isPending,
long lockedPendingNodeId,
String cleanAction)
throws DatabaseException {
/* Status variables are used to generate debug tracing info. */
boolean obsolete = false; // The LN is no longer in use.
boolean migrated = false; // The LN was in use and is migrated.
boolean lockDenied = false; // The LN lock was denied.
boolean completed = false; // This method completed.
boolean clearTarget = false; // Node was non-resident when called.
/*
* If wasCleaned is false we don't count statistics unless we migrate
* the LN. This avoids double counting.
*/
BasicLocker locker = null;
LN ln = null;
DIN parentDIN = null; // for DupCountLNs
try {
/*
* Fetch the node, if necessary. If it was not resident and it is
* an evictable LN, we will clear it after we migrate it.
*/
Node node = null;
if (!bin.isEntryKnownDeleted(index)) {
node = bin.getTarget(index);
if (node == null) {
/* If fetchTarget returns null, a deleted LN was cleaned.*/
node = bin.fetchTarget(index);
clearTarget = node instanceof LN &&
!db.getId().equals(DbTree.ID_DB_ID);
}
}
/* Don't migrate knownDeleted or deleted cleaned LNs. */
if (node == null) {
if (wasCleaned) {
nLNsDead++;
}
obsolete = true;
completed = true;
return;
}
/* Determine whether this is a DupCountLN or a regular LN. */
boolean lnIsDupCountLN = node.containsDuplicates();
if (lnIsDupCountLN) {
parentDIN = (DIN) node;
parentDIN.latch(false);
ChildReference dclRef = parentDIN.getDupCountLNRef();
lsn = dclRef.getLsn();
ln = (LN) dclRef.fetchTarget(db, parentDIN);
} else {
ln = (LN) node;
}
/*
* Get a non-blocking read lock on the LN. A pending node is
* already locked, but that node ID may be different than the
* current LN's node if a slot is reused. We must lock the current
* node to guard against aborts.
*/
if (lockedPendingNodeId != ln.getNodeId()) {
locker = new BasicLocker(env);
LockGrantType lock = locker.nonBlockingReadLock
(ln.getNodeId(), db);
if (lock == LockGrantType.DENIED) {
/*
* LN is currently locked by another Locker, so we can't
* assume anything about the value of the LSN in the bin.
*/
if (wasCleaned) {
nLNsLocked++;
}
lockDenied = true;
completed = true;
return;
}
}
/* Don't migrate deleted LNs. */
if (ln.isDeleted()) {
assert !lnIsDupCountLN;
bin.setKnownDeletedLeaveTarget(index);
if (wasCleaned) {
nLNsDead++;
}
obsolete = true;
completed = true;
return;
}
/*
* Once we have a lock, check whether the current LSN needs to be
* migrated. There is no need to migrate it if the LSN no longer
* qualifies for cleaning. Although redundant with
* isFileCleaningInProgress, check toBeCleanedFiles first because
* it is unsynchronized.
*/
Long fileNum = new Long(DbLsn.getFileNumber(lsn));
if (!toBeCleanedFiles.contains(fileNum) &&
!fileSelector.isFileCleaningInProgress(fileNum)) {
completed = true;
if (wasCleaned) {
nLNsDead++;
}
return;
}
/* Migrate the LN. */
byte[] key = getLNMainKey(bin, index);
long newLNLsn = ln.log(env, db.getId(), key, lsn, locker);
if (lnIsDupCountLN) {
parentDIN.updateDupCountLNRef(newLNLsn);
} else {
bin.updateEntry(index, newLNLsn);
}
nLNsMigrated++;
migrated = true;
completed = true;
return;
} finally {
if (parentDIN != null) {
parentDIN.releaseLatchIfOwner();
}
if (isPending) {
if (completed && !lockDenied) {
fileSelector.removePendingLN(lockedPendingNodeId);
}
} else {
/*
* If a to-be-migrated LN was not processed successfully, we
* must guarantee that the file will not be deleted and that we
* will retry the LN later. The retry information must be
* complete or we may delete a file later without processing
* all of its LNs.
*/
if (bin.getMigrate(index) && (!completed || lockDenied)) {
byte[] key = getLNMainKey(bin, index);
byte[] dupKey = getLNDupKey(bin, index, ln);
fileSelector.addPendingLN(ln, db.getId(), key, dupKey);
/* Wake up the cleaner thread to process pending LNs. */
if (!isRunning()) {
env.getUtilizationTracker().activateCleaner();
}
/*
* If we need to retry, don't clear the target since we
* would only have to fetch it again soon.
*/
clearTarget = false;
}
}
/*
* Always clear the migrate flag. If the LN could not be locked
* and the migrate flag was set, the LN will have been added to the
* pending LN set above.
*/
bin.setMigrate(index, false);
/*
* If the node was originally non-resident, clear it now so that we
* don't create more work for the evictor and reduce the cache
* memory available to the application.
*/
if (clearTarget) {
bin.updateEntry(index, null);
}
if (locker != null) {
locker.operationEnd();
}
trace(detailedTraceLevel, cleanAction, ln, lsn,
completed, obsolete, migrated);
}
}
/**
* Returns the main key for a given BIN entry.
*/
private byte[] getLNMainKey(BIN bin, int index)
throws DatabaseException {
if (bin.containsDuplicates()) {
return bin.getDupKey();
} else {
return bin.getKey(index);
}
}
/**
* Returns the duplicate key for a given BIN entry.
*/
private byte[] getLNDupKey(BIN bin, int index, LN ln)
throws DatabaseException {
DatabaseImpl db = bin.getDatabase();
if (!db.getSortedDuplicates()) {
/* The dup key is not needed for a non-duplicate DB. */
return null;
} else if (bin.containsDuplicates()) {
/* The DBIN entry key is the dup key. */
return bin.getKey(index);
} else {
/*
* The data is the dup key if the LN is not deleted. If the LN is
* deleted, this method will return null and we will do a node ID
* search later when processing the pending LN.
*/
return ln.getData();
}
}
/**
* If an IN is still in use in the in-memory tree, dirty it. The checkpoint
* invoked at the end of the cleaning run will end up rewriting it.
*/
private void processIN(IN inClone, DatabaseImpl db, long lsn)
throws DatabaseException {
boolean obsolete = false;
boolean dirtied = false;
boolean completed = false;
try {
nINsCleanedThisRun++;
if (db == null || db.getIsDeleted()) {
obsolete = true;
completed = true;
return;
}
Tree tree = db.getTree();
assert tree != null;
IN inInTree = findINInTree(tree, db, inClone, lsn);
if (inInTree == null) {
/* IN is no longer in the tree. Do nothing. */
nINsDeadThisRun++;
obsolete = true;
} else {
/*
* IN is still in the tree. Dirty it. Checkpoint will write
* it out.
*/
nINsMigratedThisRun++;
inInTree.setDirty(true);
inInTree.setCleanedSinceLastLog();
inInTree.releaseLatch();
dirtied = true;
}
completed = true;
} finally {
trace(detailedTraceLevel, CLEAN_IN, inClone, lsn,
completed, obsolete, dirtied);
}
}
/**
* Given a clone of an IN that has been taken out of the log, try to find
* it in the tree and verify that it is the current one in the log.
* Returns the node in the tree if it is found and it is current re: LSN's.
* Otherwise returns null if the clone is not found in the tree or it's not
* the latest version. Caller is responsible for unlatching the returned
* IN.
*/
private IN findINInTree(Tree tree, DatabaseImpl db, IN inClone, long lsn)
throws DatabaseException {
/* Check if inClone is the root. */
if (inClone.isDbRoot()) {
IN rootIN = isRoot(tree, db, inClone, lsn);
if (rootIN == null) {
/*
* inClone is a root, but no longer in use. Return now, because
* a call to tree.getParentNode will return something
* unexpected since it will try to find a parent.
*/
return null;
} else {
return rootIN;
}
}
/* It's not the root. Can we find it, and if so, is it current? */
inClone.latch(false);
SearchResult result = null;
try {
result = tree.getParentINForChildIN
(inClone,
true, // requireExactMatch
false, // updateGeneration
inClone.getLevel(),
null); // trackingList
if (!result.exactParentFound) {
return null;
}
int compareVal =
DbLsn.compareTo(result.parent.getLsn(result.index), lsn);
if (compareVal > 0) {
/* Log entry is obsolete. */
return null;
} else {
/*
* Log entry is same or newer than what's in the tree. Dirty
* the IN and let checkpoint write it out.
*/
IN in = (IN) result.parent.fetchTarget(result.index);
in.latch(false);
return in;
}
} finally {
if ((result != null) && (result.exactParentFound)) {
result.parent.releaseLatch();
}
}
}
private static class RootDoWork implements WithRootLatched {
private DatabaseImpl db;
private IN inClone;
private long lsn;
RootDoWork(DatabaseImpl db, IN inClone, long lsn) {
this.db = db;
this.inClone = inClone;
this.lsn = lsn;
}
public IN doWork(ChildReference root)
throws DatabaseException {
if (root == null ||
root.fetchTarget(db, null).getNodeId() !=
inClone.getNodeId()) {
return null;
}
if (DbLsn.compareTo(root.getLsn(), lsn) <= 0) {
IN rootIN = (IN) root.fetchTarget(db, null);
rootIN.latch(false);
return rootIN;
} else {
return null;
}
}
}
/**
* Check if the cloned IN is the same node as the root in tree. Return the
* real root if it is, null otherwise. If non-null is returned, the
* returned IN (the root) is latched -- caller is responsible for
* unlatching it.
*/
private IN isRoot(Tree tree, DatabaseImpl db, IN inClone, long lsn)
throws DatabaseException {
RootDoWork rdw = new RootDoWork(db, inClone, lsn);
return tree.withRootLatched(rdw);
}
/**
* Reset per-run counters.
*/
private void resetPerRunCounters() {
nINsObsoleteThisRun = 0;
nINsCleanedThisRun = 0;
nINsDeadThisRun = 0;
nINsMigratedThisRun = 0;
nLNsObsoleteThisRun = 0;
nLNsCleanedThisRun = 0;
nLNsDeadThisRun = 0;
nLNsMigratedThisRun = 0;
nLNsMarkedThisRun = 0;
nLNsLockedThisRun = 0;
nEntriesReadThisRun = 0;
nRepeatIteratorReadsThisRun = 0;
}
private void accumulatePerRunCounters() {
nINsObsolete += nINsObsoleteThisRun;
nINsCleaned += nINsCleanedThisRun;
nINsDead += nINsDeadThisRun;
nINsMigrated += nINsMigratedThisRun;
nLNsObsolete += nLNsObsoleteThisRun;
nLNsCleaned += nLNsCleanedThisRun;
nLNsDead += nLNsDeadThisRun;
nLNsMigrated += nLNsMigratedThisRun;
nLNsMarked += nLNsMarkedThisRun;
nLNsLocked += nLNsLockedThisRun;
nRepeatIteratorReads += nRepeatIteratorReadsThisRun;
}
/**
* Send trace messages to the java.util.logger. Don't rely on the logger
* alone to conditionalize whether we send this message, we don't even want
* to construct the message if the level is not enabled.
*/
private void trace(Level level,
String action,
Node node,
long logLsn,
boolean completed,
boolean obsolete,
boolean dirtiedMigrated) {
Logger logger = env.getLogger();
if (logger.isLoggable(level)) {
StringBuffer sb = new StringBuffer();
sb.append(action);
if (node != null) {
sb.append(" node=");
sb.append(node.getNodeId());
}
sb.append(" logLsn=");
sb.append(DbLsn.getNoFormatString(logLsn));
sb.append(" complete=").append(completed);
sb.append(" obsolete=").append(obsolete);
sb.append(" dirtiedOrMigrated=").append(dirtiedMigrated);
logger.log(level, sb.toString());
}
}
}