/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.jackrabbit.core.data;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import javax.jcr.RepositoryException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.LazyFileInputStream;
import org.apache.jackrabbit.util.TransientFileFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class implements a LRU cache used by {@link CachingDataStore}. If cache
* size exceeds limit, this cache goes in purge mode. In purge mode any
* operation to cache is no-op. After purge cache size would be less than
* cachePurgeResizeFactor * maximum size.
*/
public class LocalCache {
/**
* Logger instance.
*/
static final Logger LOG = LoggerFactory.getLogger(LocalCache.class);
/**
* The file names of the files that need to be deleted.
*/
final Set<String> toBeDeleted = new HashSet<String>();
/**
* The filename Vs file size LRU cache.
*/
LRUCache cache;
/**
* The directory where the files are created.
*/
private final File directory;
/**
* The directory where tmp files are created.
*/
private final File tmp;
/**
* The maximum size of cache in bytes.
*/
private long maxSize;
/**
* If true cache is in purgeMode and not available. All operation would be
* no-op.
*/
private volatile boolean purgeMode;
private AsyncUploadCache asyncUploadCache;
/**
* Build LRU cache of files located at 'path'. It uses lastModified property
* of file to build LRU cache. If cache size exceeds limit size, this cache
* goes in purge mode. In purge mode any operation to cache is no-op.
*
* @param path file system path
* @param tmpPath temporary directory used by cache.
* @param maxSize maximum size of cache.
* @param cachePurgeTrigFactor factor which triggers cache to purge mode.
* That is if current size exceed (cachePurgeTrigFactor * maxSize), the
* cache will go in auto-purge mode.
* @param cachePurgeResizeFactor after cache purge size of cache will be
* just less (cachePurgeResizeFactor * maxSize).
* @param asyncUploadCache {@link AsyncUploadCache}
* @throws RepositoryException
*/
public LocalCache(String path, String tmpPath, long size, double cachePurgeTrigFactor,
double cachePurgeResizeFactor, AsyncUploadCache asyncUploadCache) throws IOException,
ClassNotFoundException {
this.maxSize = size;
directory = new File(path);
tmp = new File(tmpPath);
LOG.info("cachePurgeTrigFactor = " + cachePurgeTrigFactor + ", cachePurgeResizeFactor = " + cachePurgeResizeFactor
+ ", cachePurgeTrigFactorSize = " + (cachePurgeTrigFactor * size) + ", cachePurgeResizeFactor = "
+ (cachePurgeResizeFactor * size));
cache = new LRUCache(size, cachePurgeTrigFactor, cachePurgeResizeFactor);
this.asyncUploadCache = asyncUploadCache;
new Thread(new CacheBuildJob()).start();
}
/**
* Store an item in the cache and return the input stream. If cache is in
* purgeMode or file doesn't exists, inputstream from a
* {@link TransientFileFactory#createTransientFile(String, String, File)} is
* returned. Otherwise inputStream from cached file is returned. This method
* doesn't close the incoming inputstream.
*
* @param fileName the key of cache.
* @param in {@link InputStream}
* @return the (new) input stream.
*/
public InputStream store(String fileName, final InputStream in)
throws IOException {
fileName = fileName.replace("\\", "/");
File f = getFile(fileName);
long length = 0;
synchronized (this) {
if (!f.exists() || isInPurgeMode()) {
OutputStream out = null;
File transFile = null;
try {
TransientFileFactory tff = TransientFileFactory.getInstance();
transFile = tff.createTransientFile("s3-", "tmp", tmp);
out = new BufferedOutputStream(new FileOutputStream(transFile));
length = IOUtils.copyLarge(in, out);
} finally {
IOUtils.closeQuietly(out);
}
// rename the file to local fs cache
if (canAdmitFile(length)
&& (f.getParentFile().exists() || f.getParentFile().mkdirs())
&& transFile.renameTo(f) && f.exists()) {
if (transFile.exists() && transFile.delete()) {
LOG.info("tmp file = " + transFile.getAbsolutePath()
+ " not deleted successfully");
}
transFile = null;
if (LOG.isDebugEnabled()) {
LOG.debug("file [" + fileName + "] added to local cache.");
}
cache.put(fileName, f.length());
} else {
f = transFile;
}
} else {
// f.exists and not in purge mode
f.setLastModified(System.currentTimeMillis());
cache.put(fileName, f.length());
}
cache.tryPurge();
return new LazyFileInputStream(f);
}
}
/**
* Store an item along with file in cache. Cache size is increased by
* {@link File#length()} If file already exists in cache,
* {@link File#setLastModified(long)} is updated with current time.
*
* @param fileName the key of cache.
* @param src file to be added to cache.
* @throws IOException
*/
public synchronized File store(String fileName, final File src) {
try {
return store(fileName, src, false).getFile();
} catch (IOException ioe) {
LOG.warn("Exception in addding file [" + fileName + "] to local cache.", ioe);
}
return null;
}
/**
* This method add file to {@link LocalCache} and tries that file can be
* added to {@link AsyncUploadCache}. If file is added to
* {@link AsyncUploadCache} successfully, it sets
* {@link AsyncUploadResult#setAsyncUpload(boolean)} to true.
*
* @param fileName name of the file.
* @param src source file.
* @param tryForAsyncUpload If true it tries to add fileName to
* {@link AsyncUploadCache}
* @return {@link AsyncUploadCacheResult}. This method sets
* {@link AsyncUploadResult#setAsyncUpload(boolean)} to true, if
* fileName is added to {@link AsyncUploadCache} successfully else
* it sets {@link AsyncUploadCacheResult#setAsyncUpload(boolean)} to
* false. {@link AsyncUploadCacheResult#getFile()} contains cached
* file, if it is added to {@link LocalCache} or original file.
* @throws IOException
*/
public synchronized AsyncUploadCacheResult store(String fileName, File src, boolean tryForAsyncUpload) throws IOException {
fileName = fileName.replace("\\", "/");
File dest = getFile(fileName);
File parent = dest.getParentFile();
AsyncUploadCacheResult result = new AsyncUploadCacheResult();
result.setFile(src);
result.setAsyncUpload(false);
boolean destExists = false;
if ((destExists = dest.exists())
|| (src.exists() && !dest.exists() && !src.equals(dest) && canAdmitFile(src.length())
&& (parent.exists() || parent.mkdirs()) && (src.renameTo(dest)))) {
if (destExists) {
dest.setLastModified(System.currentTimeMillis());
}
cache.put(fileName, dest.length());
if (LOG.isDebugEnabled()) {
LOG.debug("file [" + fileName + "] added to local cache.");
}
result.setFile(dest);
if (tryForAsyncUpload) {
result.setAsyncUpload(asyncUploadCache.add(fileName).canAsyncUpload());
}
}
cache.tryPurge();
return result;
}
/**
* Return the inputstream from from cache, or null if not in the cache.
*
* @param fileName name of file.
* @return stream or null.
*/
public InputStream getIfStored(String fileName) throws IOException {
File file = getFileIfStored(fileName);
return file == null ? null : new LazyFileInputStream(file);
}
public synchronized File getFileIfStored(String fileName) throws IOException {
fileName = fileName.replace("\\", "/");
File f = getFile(fileName);
// return file in purge mode = true and file present in asyncUploadCache
// as asyncUploadCache's files will be not be deleted in cache purge.
if (!f.exists() || (isInPurgeMode() && !asyncUploadCache.hasEntry(fileName, false))) {
log("purgeMode true or file doesn't exists: getFileIfStored returned");
return null;
} else {
// touch entry in LRU caches
cache.put(fileName, f.length());
f.setLastModified(System.currentTimeMillis());
return f;
}
}
/**
* Delete file from cache. Size of cache is reduced by file length. The
* method is no-op if file doesn't exist in cache.
*
* @param fileName file name that need to be removed from cache.
*/
public synchronized void delete(String fileName) {
if (isInPurgeMode()) {
log("purgeMode true :delete returned");
return;
}
fileName = fileName.replace("\\", "/");
cache.remove(fileName);
}
/**
* Returns length of file if exists in cache else returns null.
* @param fileName name of the file.
*/
public synchronized Long getFileLength(String fileName) {
Long length = null;
try {
length = cache.get(fileName);
if( length == null ) {
File f = getFileIfStored(fileName);
if (f != null) {
length = f.length();
}
}
} catch (IOException ignore) {
}
return length;
}
/**
* Close the cache. Cache maintain set of files which it was not able to
* delete successfully. This method will an attempt to delete all
* unsuccessful delete files.
*/
public void close() {
log("close");
deleteOldFiles();
}
/**
* Check if cache can admit file of given length.
* @param length of the file.
* @return true if yes else return false.
*/
private synchronized boolean canAdmitFile(final long length) {
//order is important here
boolean value = !isInPurgeMode() && (cache.canAdmitFile(length));
if (!value) {
log("cannot admit file of length=" + length + " and currentSizeInBytes=" + cache.currentSizeInBytes);
}
return value;
}
/**
* Return true if cache is in purge mode else return false.
*/
synchronized boolean isInPurgeMode() {
return purgeMode || maxSize == 0;
}
/**
* Set purge mode. If set to true all cache operation will be no-op. If set
* to false, all operations to cache are available.
*
* @param purgeMode purge mode
*/
synchronized void setPurgeMode(final boolean purgeMode) {
this.purgeMode = purgeMode;
}
File getFile(final String fileName) {
return new File(directory, fileName);
}
private void deleteOldFiles() {
int initialSize = toBeDeleted.size();
int count = 0;
for (String n : new ArrayList<String>(toBeDeleted)) {
if (tryDelete(n)) {
count++;
}
}
LOG.info("deleted [" + count + "]/[" + initialSize + "] files");
}
/**
* This method tries to delete a file. If it is not able to delete file due
* to any reason, it add it toBeDeleted list.
*
* @param fileName name of the file which will be deleted.
* @return true if this method deletes file successfuly else return false.
*/
boolean tryDelete(final String fileName) {
log("cache delete " + fileName);
File f = getFile(fileName);
if (f.exists() && f.delete()) {
log(fileName + " deleted successfully");
toBeDeleted.remove(fileName);
while (true) {
f = f.getParentFile();
if (f.equals(directory) || f.list().length > 0) {
break;
}
// delete empty parent folders (except the main directory)
f.delete();
}
return true;
} else if (f.exists()) {
LOG.info("not able to delete file = " + f.getAbsolutePath());
toBeDeleted.add(fileName);
return false;
}
return true;
}
static int maxSizeElements(final long bytes) {
// after a CQ installation, the average item in
// the data store is about 52 KB
int count = (int) (bytes / 65535);
count = Math.max(1024, count);
count = Math.min(64 * 1024, count);
return count;
}
static void log(final String s) {
LOG.debug(s);
}
/**
* A LRU based extension {@link LinkedHashMap}. The key is file name and
* value is length of file.
*/
private class LRUCache extends LinkedHashMap<String, Long> {
private static final long serialVersionUID = 1L;
volatile long currentSizeInBytes;
final long maxSizeInBytes;
final long cachePurgeResize;
final long cachePurgeTrigSize;
LRUCache(final long maxSizeInBytes,
final double cachePurgeTrigFactor,
final double cachePurgeResizeFactor) {
super(maxSizeElements(maxSizeInBytes), (float) 0.75, true);
this.maxSizeInBytes = maxSizeInBytes;
this.cachePurgeTrigSize = new Double(cachePurgeTrigFactor
* maxSizeInBytes).longValue();
this.cachePurgeResize = new Double(cachePurgeResizeFactor
* maxSizeInBytes).longValue();
}
/**
* Overridden {@link Map#remove(Object)} to delete corresponding file
* from file system.
*/
@Override
public synchronized Long remove(final Object key) {
String fileName = (String) key;
fileName = fileName.replace("\\", "/");
try {
// not removing file from local cache, if there is in progress
// async upload on it.
if (asyncUploadCache.hasEntry(fileName, false)) {
LOG.info("AsyncUploadCache upload contains file [" + fileName
+ "]. Not removing it from LocalCache.");
return null;
}
} catch (IOException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("error: ", e);
}
return null;
}
Long flength = null;
if (tryDelete(fileName)) {
flength = super.remove(key);
if (flength != null) {
log("cache entry { " + fileName + "} with size {" + flength + "} removed.");
currentSizeInBytes -= flength.longValue();
}
} else if (!getFile(fileName).exists()) {
// second attempt. remove from cache if file doesn't exists
flength = super.remove(key);
if (flength != null) {
log(" file not exists. cache entry { " + fileName + "} with size {" + flength + "} removed.");
currentSizeInBytes -= flength.longValue();
}
}
return flength;
}
@Override
public synchronized Long put(final String fileName, final Long value) {
Long oldValue = cache.get(fileName);
if (oldValue == null) {
long flength = value.longValue();
currentSizeInBytes += flength;
return super.put(fileName.replace("\\", "/"), value);
}
toBeDeleted.remove(fileName);
return oldValue;
}
/**
* This method tries purging of local cache. It checks if local cache
* has exceeded the defined limit then it triggers purge cache job in a
* seperate thread.
*/
synchronized void tryPurge() {
if (currentSizeInBytes > cachePurgeTrigSize && !isInPurgeMode()) {
setPurgeMode(true);
LOG.info("currentSizeInBytes[" + cache.currentSizeInBytes + "] exceeds (cachePurgeTrigSize)[" + cache.cachePurgeTrigSize
+ "]");
new Thread(new PurgeJob()).start();
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("currentSizeInBytes[" + cache.currentSizeInBytes + "] and (cachePurgeTrigSize)[" + cache.cachePurgeTrigSize
+ "], isInPurgeMode =[" + isInPurgeMode() + "]");
}
}
}
/**
* This method check if cache can admit file of given length.
* @param length length of file.
* @return true if cache size + length is less than maxSize.
*/
synchronized boolean canAdmitFile(final long length) {
return cache.currentSizeInBytes + length < cache.maxSizeInBytes;
}
}
/**
* This class performs purging of local cache. It implements
* {@link Runnable} and should be invoked in a separate thread.
*/
private class PurgeJob implements Runnable {
public PurgeJob() {
// TODO Auto-generated constructor stub
}
/**
* This method purges local cache till its size is less than
* cacheResizefactor * maxSize
*/
@Override
public void run() {
try {
synchronized (cache) {
LOG.info(" cache purge job started");
// first try to delete toBeDeleted files
int initialSize = cache.size();
for (String fileName : new ArrayList<String>(toBeDeleted)) {
cache.remove(fileName);
}
Iterator<Map.Entry<String, Long>> itr = cache.entrySet().iterator();
while (itr.hasNext()) {
Map.Entry<String, Long> entry = itr.next();
if (entry.getKey() != null) {
if (cache.currentSizeInBytes > cache.cachePurgeResize) {
itr.remove();
} else {
break;
}
}
}
LOG.info(" cache purge job completed: cleaned ["
+ (initialSize - cache.size())
+ "] files and currentSizeInBytes = [ "
+ cache.currentSizeInBytes + "]");
}
} catch (Exception e) {
LOG.error("error in purge jobs:", e);
} finally {
setPurgeMode(false);
}
}
}
/**
* This class implements {@link Runnable} interface to build LRU cache
* asynchronously.
*/
private class CacheBuildJob implements Runnable {
public void run() {
long startTime = System.currentTimeMillis();
ArrayList<File> allFiles = new ArrayList<File>();
Iterator<File> it = FileUtils.iterateFiles(directory, null, true);
while (it.hasNext()) {
File f = it.next();
allFiles.add(f);
}
long t1 = System.currentTimeMillis();
if (LOG.isDebugEnabled()) {
LOG.debug("Time taken to recursive [" + allFiles.size() + "] took [" + ((t1 - startTime) / 1000) + "]sec");
}
Collections.sort(allFiles, new Comparator<File>() {
public int compare(File o1, File o2) {
long l1 = o1.lastModified(), l2 = o2.lastModified();
return l1 < l2 ? -1 : l1 > l2 ? 1 : 0;
}
});
long t2 = System.currentTimeMillis();
if (LOG.isDebugEnabled()) {
LOG.debug("Time taken to sort [" + allFiles.size() + "] took [" + ((t2 - t1) / 1000) + "]sec");
}
String dataStorePath = directory.getAbsolutePath();
long time = System.currentTimeMillis();
int count = 0;
for (File f : allFiles) {
if (f.exists()) {
count++;
String name = f.getPath();
if (name.startsWith(dataStorePath)) {
name = name.substring(dataStorePath.length());
}
// convert to java path format
name = name.replace("\\", "/");
if (name.startsWith("/") || name.startsWith("\\")) {
name = name.substring(1);
}
store(name, f);
long now = System.currentTimeMillis();
if (now > time + 10000) {
LOG.info("Processed {" + (count) + "}/{" + allFiles.size() + "}");
time = now;
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Processed {" + count + "}/{" + allFiles.size() + "} , currentSizeInBytes = " + cache.currentSizeInBytes
+ ", maxSizeInBytes = " + cache.maxSizeInBytes + ", cache.filecount = " + cache.size());
}
long t3 = System.currentTimeMillis();
LOG.info("Time to build cache of [" + allFiles.size() + "] took [" + ((t3 - startTime) / 1000) + "]sec");
}
}
}