/* Copyright (c) 2008 TOPP - www.openplans.org. All rights reserved.
* This code is licensed under the GPL 2.0 license, availible at the root
* application directory.
*/
package org.geoserver.rest.util;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.io.Reader;
import java.net.URL;
import java.nio.channels.Channel;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import javax.xml.transform.stream.StreamSource;
import org.apache.commons.io.FilenameUtils;
import org.geotools.data.DataUtilities;
/**
* Assorted IO related utilities
*
* @author Simone Giannecchini, GeoSolutions SAS
*
*/
public class IOUtils extends org.apache.commons.io.IOUtils {
private final static Logger LOGGER = org.geotools.util.logging.Logging.getLogger(FileCleaner.class);
/** Default size of element for {@link FileChannel} based copy method.*/
private static final int DEFAULT_SIZE = 10 * 1024 * 1024;
/**Background to perform file deletions.*/
private final static FileCleaner FILE_CLEANER = new FileCleaner();
private final static Set<String> FILES_PATH = Collections.synchronizedSet(new HashSet<String>());
private final static Map<String, Integer> FILE_ATTEMPTS_COUNTS = Collections.synchronizedMap(new HashMap<String, Integer>());
/**
* 30 seconds is the default period beteen two checks.
*/
private static long DEFAULT_PERIOD = 5L;
/**
* The default number of attempts is 50
*/
private final static int DEF_MAX_ATTEMPTS = 50;
static {
FILE_CLEANER.setMaxAttempts(100);
FILE_CLEANER.setPeriod(30);
FILE_CLEANER.setPriority(1);
FILE_CLEANER.start();
}
/**
* Simple class implementing a periodic Thread that periodically tries to
* delete the files that were provided to him.
* <p>
* It tries to delete each file at most {@link FileCleaner#maxAttempts}
* number of times. If this number is exceeded it simply throws the file
* away notifying the users with a warning message.
*
* @author Simone Giannecchini, GeoSolutions.
*/
public final static class FileCleaner extends Thread {
/**
* Maximum number of attempts to delete a given {@link File}.
*
* <p>
* If the provided number of attempts is exceeded we simply drop warn the
* user and we remove the {@link File} from our list.
*/
private int maxAttempts = DEF_MAX_ATTEMPTS;
/**
* Period in seconds between two checks.
*/
private volatile long period = DEFAULT_PERIOD;
/**
* Asks this {@link FileCleaner} to clean up this file.
*
* @param fileToDelete {@link File} that we want to permanently delete.
*/
public void addFile(final File fileToDelete) {
// does it exists
if (!fileToDelete.exists())
return;
synchronized (FILES_PATH) {
synchronized (FILE_ATTEMPTS_COUNTS) {
// /////////////////////////////////////////////////////////////////
//
// We add the file to our lists for later check.
//
// /////////////////////////////////////////////////////////////////
if (!FILES_PATH.contains(fileToDelete.getAbsolutePath())) {
FILES_PATH.add(fileToDelete.getAbsolutePath());
FILE_ATTEMPTS_COUNTS.put(fileToDelete.getAbsolutePath(),
new Integer(0));
}
}
}
}
/**
* Default constructor for a {@link FileCleaner}.
*/
public FileCleaner() {
this(DEFAULT_PERIOD, Thread.NORM_PRIORITY - 3, DEF_MAX_ATTEMPTS);
}
/**
* Constructor for a {@link FileCleaner}.
*
* @param period default time period between two cycles.
* @param priority is the priority for the cleaner thread.
* @param maxattempts maximum number of time the cleaner thread tries to delete a file.
*/
public FileCleaner(long period, int priority, int maxattempts) {
this.period = period;
this.setName("FileCleaner");
this.setPriority(priority);
this.setDaemon(true);
this.maxAttempts = maxattempts;
}
/**
* This method does the magic:
*
* <ol>
* <li>iterate over all the files</li>
* <li>try to delete it</li>
* <li>if successful drop the file references</li>
* <li>if not successful increase the attempts count for the file and call
* the gc. If the maximum number was exceeded drop the file and warn the
* user </li>
*
*/
public void run() {
while (true) {
try {
synchronized (FILES_PATH) {
synchronized (FILE_ATTEMPTS_COUNTS) {
final Iterator<String> it = FILES_PATH.iterator();
while (it.hasNext()) {
// get next file path and its count
final String sFile = it.next();
if(LOGGER.isLoggable(Level.INFO))
LOGGER.info("Trying to remove file " + sFile);
int attempts = FILE_ATTEMPTS_COUNTS.get(sFile).intValue();
if (!new File(sFile).exists()) {
it.remove();
FILE_ATTEMPTS_COUNTS.remove(sFile);
} else {
// try to delete it
if (new File(sFile).delete()) {
if(LOGGER.isLoggable(Level.INFO))
LOGGER.info("Successfully removed file "+ sFile);
it.remove();
FILE_ATTEMPTS_COUNTS.remove(sFile);
} else {
if(LOGGER.isLoggable(Level.INFO))
LOGGER.info("Unable to remove file "
+ sFile);
attempts++;
if (maxAttempts < attempts) {
if(LOGGER.isLoggable(Level.INFO))
LOGGER.info("Dropping file " + sFile);
it.remove();
FILE_ATTEMPTS_COUNTS.remove(sFile);
if (LOGGER.isLoggable(Level.WARNING))
LOGGER
.warning("Unable to delete file "
+ sFile);
} else {
FILE_ATTEMPTS_COUNTS.remove(sFile);
FILE_ATTEMPTS_COUNTS.put(sFile,new Integer(attempts));
// might help, see
// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4715154
Runtime.getRuntime().gc();
Runtime.getRuntime().gc();
Runtime.getRuntime().gc();
Runtime.getRuntime().gc();
Runtime.getRuntime().gc();
Runtime.getRuntime().gc();
System.runFinalization();
System.runFinalization();
System.runFinalization();
System.runFinalization();
System.runFinalization();
System.runFinalization();
}
}
}
}
}
}
Thread.sleep(period * 1000);
} catch (Throwable t) {
if(LOGGER.isLoggable(Level.INFO))
LOGGER.log(Level.INFO, t.getLocalizedMessage(), t);
}
}
}
/**
* Retrieves the maximum number of times we try to delete a file before giving up.
* @return the maximum number of times we try to delete a file before giving up.
*
*/
public int getMaxAttempts() {
synchronized (FILES_PATH) {
synchronized (FILE_ATTEMPTS_COUNTS) {
return maxAttempts;
}
}
}
/**
* Sets the maximum number of times we try to delete a file before giving up.
* @param maxAttempts the maximum number of times we try to delete a file before giving up.
*
*/
public void setMaxAttempts(int maxAttempts) {
synchronized (FILES_PATH) {
synchronized (FILE_ATTEMPTS_COUNTS) {
this.maxAttempts = maxAttempts;
}
}
}
/**
* Retrieves the period in seconds for this {@link FileCleaner} .
* @return the period in seconds for this {@link FileCleaner} .
*
*/
public long getPeriod() {
return period;
}
/**
* Sets the period in seconds for this {@link FileCleaner} .
* @param period the new period for this {@link FileCleaner} .
*
*/
public void setPeriod(long period) {
this.period = period;
}
}
/**
* Copies the content of the source channel onto the destination channel.
*
* @param bufferSize size of the temp buffer to use for this copy.
* @param source the source {@link ReadableByteChannel}.
* @param destination the destination {@link WritableByteChannel};.
* @throws IOException in case something bad happens.
*/
public static void copyChannel(int bufferSize, ReadableByteChannel source,WritableByteChannel destination) throws IOException {
inputNotNull(source,destination);
if(!source.isOpen()||!destination.isOpen())
throw new IllegalStateException("Source and destination channels must be open.");
final java.nio.ByteBuffer buffer= java.nio.ByteBuffer.allocateDirect(bufferSize);
while(source.read(buffer)!=-1)
{
//prepare the buffer for draining
buffer.flip();
//write to destination
while(buffer.hasRemaining())
destination.write(buffer);
//clear
buffer.clear();
}
}
/**
* Optimize version of copy method for file channels.
*
* @param bufferSize size of the temp buffer to use for this copy.
* @param source the source {@link ReadableByteChannel}.
* @param destination the destination {@link WritableByteChannel};.
* @throws IOException in case something bad happens.
*/
public static void copyFileChannel(int bufferSize, FileChannel source,
FileChannel destination) throws IOException {
inputNotNull(source,destination);
if(!source.isOpen()||!destination.isOpen())
throw new IllegalStateException("Source and destination channels must be open.");
FileLock lock = null;
try {
lock = destination.lock();
final long sourceSize = source.size();
long pos = 0;
while (pos < sourceSize) {
// read and flip
final long remaining = (sourceSize - pos);
final int mappedZoneSize = remaining >= bufferSize ? bufferSize
: (int) remaining;
destination.transferFrom(source, pos, mappedZoneSize);
// update zone
pos += mappedZoneSize;
}
} finally {
if (lock != null) {
try {
lock.release();
}catch (Throwable t) {
if(LOGGER.isLoggable(Level.INFO))
LOGGER.log(Level.INFO,t.getLocalizedMessage(),t);
}
}
}
}
/**
* Close the specified input <code>FileChannel</code>
*
* @throws IOException in case something bad happens.
*/
public static void closeQuietly(Channel channel)
throws IOException {
inputNotNull(channel);
if (channel.isOpen())
channel.close();
}
/**
* Checks if the input is not null.
* @param oList list of elements to check for null.
*/
private static void inputNotNull(Object...oList) {
for(Object o: oList)
if(o==null)
throw new NullPointerException("Input objects cannot be null");
}
/**
* Copy the input file onto the output file using a default buffer size.
*
* @param sourceFile the {@link File} to copy from.
* @param destinationFile the {@link File} to copy to.
* @throws IOException in case something bad happens.
*/
public static void copyFile(File sourceFile, File destinationFile)
throws IOException {
copyFile(sourceFile, destinationFile, DEFAULT_SIZE);
}
/**
* Copy the input file onto the output file using the specified buffer size.
*
* @param sourceFile the {@link File} to copy from.
* @param destinationFile the {@link File} to copy to.
* @param size buffer size.
* @throws IOException in case something bad happens.
*/
public static void copyFile(File sourceFile, File destinationFile, int size)
throws IOException {
inputNotNull(sourceFile,destinationFile);
if(!sourceFile.exists()||!sourceFile.canRead()||!sourceFile.isFile())
throw new IllegalStateException("Source is not in a legal state.");
if (!destinationFile.exists()) {
destinationFile.createNewFile();
}
if (destinationFile.getAbsolutePath().equalsIgnoreCase(
sourceFile.getAbsolutePath()))
throw new IllegalArgumentException("Cannot copy a file on itself");
FileChannel source = null;
FileChannel destination = null;
source = new RandomAccessFile(sourceFile, "r").getChannel();
destination = new RandomAccessFile(destinationFile, "rw").getChannel();
try {
copyFileChannel(size, source, destination);
} finally {
try {
if (source != null) {
try {
source.close();
}
catch (Throwable t) {
if(LOGGER.isLoggable(Level.INFO))
LOGGER.log(Level.INFO,t.getLocalizedMessage(),t);
}
}
} finally {
if (destination != null) {
try {
destination.close();
}
catch (Throwable t) {
if(LOGGER.isLoggable(Level.INFO))
LOGGER.log(Level.INFO,t.getLocalizedMessage(),t);
}
}
}
}
}
/**
* Delete all the files with matching the specified {@link FilenameFilter} in the specified directory.
* The method can work recursively.
*
* @param sourceDirectory the directory to delete files from.
* @param filter the {@link FilenameFilter} to use for selecting files to delete.
* @param recursive boolean that specifies if we want to delete files recursively or not.
* @return
*/
public static boolean deleteDirectory(File sourceDirectory, FilenameFilter filter, boolean recursive, boolean deleteItself) {
inputNotNull(sourceDirectory,filter);
if(!sourceDirectory.exists()||!sourceDirectory.canRead()||!sourceDirectory.isDirectory())
throw new IllegalStateException("Source is not in a legal state.");
final File[] files = (filter != null ? sourceDirectory.listFiles(filter) : sourceDirectory.listFiles());
for (File file:files) {
if (file.isDirectory()) {
if(recursive)
deleteDirectory(file, filter,recursive,deleteItself);
} else {
if(!file.delete())
return false;
}
}
return deleteItself?sourceDirectory.delete():true;
}
/**
* Delete the specified File.
*
* @param sourceDirectory the directory to delete files from.
* @param filter the {@link FilenameFilter} to use for selecting files to delete.
* @param recursive boolean that specifies if we want to delete files recursively or not.
* @return
*/
public static void deleteFile(File file) {
inputNotNull(file);
if(!file.exists()||!file.canRead()||!file.isFile())
throw new IllegalStateException("Source is not in a legal state.");
if(file.delete())
return;
IOUtils.FILE_CLEANER.addFile(file);
}
/**
* Get an input <code>FileChannel</code> for the provided
* <code>File</code>
*
* @param file
* <code>File</code> for which we need to get an input
* <code>FileChannel</code>
* @return a <code>FileChannel</code>
* @throws IOException in case something bad happens.
*/
public static FileChannel getInputChannel(File source)
throws IOException {
inputNotNull(source);
if(!source.exists()||!source.canRead()||!source.isDirectory())
throw new IllegalStateException("Source is not in a legal state.");
FileChannel channel = null;
while (channel == null) {
try {
channel = new FileInputStream(source).getChannel();
} catch (Exception e) {
channel = null;
}
}
return channel;
}
/**
* Get an output <code>FileChannel</code> for the provided
* <code>File</code>
*
* @param file
* <code>File</code> for which we need to get an output
* <code>FileChannel</code>
* @return a <code>FileChannel</code>
* @throws IOException in case something bad happens.
*/
public static FileChannel getOuputChannel(File file)
throws IOException {
inputNotNull(file);
return new RandomAccessFile(file, "rw").getChannel();
}
/**
* Move the specified input file to the specified destination directory.
*
* @param source
* the input <code>File</code> which need to be moved.
* @param destDir
* the destination directory where to move the file.
* @throws IOException
*/
public static void moveFileTo(File source, File destDir, boolean removeInputFile) throws IOException {
inputNotNull(source,destDir);
if(!source.exists()||!source.canRead()||source.isDirectory())
throw new IllegalStateException("Source is not in a legal state." );
if(!destDir.exists()||!destDir.canWrite()||!destDir.isDirectory())
throw new IllegalStateException("Source is not in a legal state." );
if (destDir.getAbsolutePath().equalsIgnoreCase(
source.getParentFile().getAbsolutePath()))
return;
// ///////////////////////////////////////////////////////////////
//
// Copy the inputFile in the specified destination directory
//
// ///////////////////////////////////////////////////////////////
copyFile(source, new File(destDir, source.getName()));
// ///////////////////////////////////////////////////////////////
//
// Delete the source file.
//
// ///////////////////////////////////////////////////////////////
// we need to call the gc, see
// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4715154
if (removeInputFile)
FILE_CLEANER.addFile(source);
}
/**
* Tries to convert a {@link URL} into a {@link File}. Return null if something bad happens
* @param fileURL {@link URL} to be converted into a {@link File}.
* @return {@link File} for this {@link URL} or null.
*/
public static File URLToFile(URL fileURL) {
inputNotNull(fileURL);
try {
final File retFile= DataUtilities.urlToFile(fileURL);
return retFile;
}catch (Throwable t) {
if(LOGGER.isLoggable(Level.FINE))
LOGGER.log(Level.FINE,t.getLocalizedMessage(),t);
}
return null;
}
/**
* Copy {@link InputStream} to {@link OutputStream}.
*
* @param sourceStream {@link InputStream} to copy from.
* @param destinationStream {@link OutputStream} to copy to.
* @param closeInput quietly close {@link InputStream}.
* @param closeOutput quietly close {@link OutputStream}
* @throws IOException in case something bad happens.
*/
public static void copyStream(InputStream sourceStream,
OutputStream destinationStream, boolean closeInput,
boolean closeOutput) throws IOException {
copyStream(sourceStream, destinationStream, DEFAULT_SIZE, closeInput,
closeOutput);
}
/**
* Copy {@link InputStream} to {@link OutputStream}.
*
* @param sourceStream {@link InputStream} to copy from.
* @param destinationStream {@link OutputStream} to copy to.
* @param size size of the buffer to use internally.
* @param closeInput quietly close {@link InputStream}.
* @param closeOutput quietly close {@link OutputStream}
* @throws IOException in case something bad happens.
*/
public static void copyStream(InputStream sourceStream,
OutputStream destinationStream, int size, boolean closeInput,
boolean closeOutput) throws IOException {
inputNotNull(sourceStream,destinationStream);
byte[] buf = new byte[size];
int n = -1;
try {
while (-1 != (n = sourceStream.read(buf))) {
destinationStream.write(buf, 0, n);
destinationStream.flush();
}
} finally {
// closing streams and connections
try {
destinationStream.flush();
} finally {
try {
if (closeOutput)
destinationStream.close();
} finally {
try {
if (closeInput)
sourceStream.close();
} finally {
}
}
}
}
}
/**
* Convert the input from the provided {@link InputStream} into a {@link String}.
*
* @param inputStream the {@link InputStream} to copy from.
* @return a {@link String} that contains the content of the provided {@link InputStream}.
* @throws IOException in case something bad happens.
*/
public static String getStringFromStream(InputStream inputStream) throws IOException {
inputNotNull(inputStream);
final Reader inReq = new InputStreamReader(inputStream);
return getStringFromReader(inReq);
}
/**
* Convert the input from the provided {@link Reader} into a {@link String}.
*
* @param inputStream the {@link Reader} to copy from.
* @return a {@link String} that contains the content of the provided {@link Reader}.
* @throws IOException in case something bad happens.
*/
public static String getStringFromReader(final Reader inputReader)
throws IOException {
inputNotNull(inputReader);
final StringBuilder sb = new StringBuilder();
final char[] buffer = new char[1024];
int len;
while ((len = inputReader.read(buffer)) >= 0) {
char[] read = new char[len];
System.arraycopy(buffer, 0, read, 0, len);
sb.append(read);
}
return sb.toString();
}
/**
* Convert the input from the provided {@link Reader} into a {@link String}.
*
* @param inputStream the {@link Reader} to copy from.
* @return a {@link String} that contains the content of the provided {@link Reader}.
* @throws IOException in case something bad happens.
*/
public static String getStringFromStreamSource(StreamSource src) throws IOException {
inputNotNull(src);
InputStream inputStream = src.getInputStream();
if(inputStream != null) {
return getStringFromStream(inputStream);
}else {
final Reader r = src.getReader();
return getStringFromReader(r);
}
}
/**
* Inflate the provided {@link ZipFile} in the provided output directory.
*
* @param archive the {@link ZipFile} to inflate.
* @param outputDirectory the directory where to inflate the archive.
* @throws IOException in case something bad happens.
* @throws FileNotFoundException in case something bad happens.
*/
public static void inflate(ZipFile archive, File outputDirectory, String fileName) throws IOException,
FileNotFoundException {
final Enumeration<? extends ZipEntry> entries = archive.entries();
try {
while (entries.hasMoreElements()) {
ZipEntry entry = (ZipEntry)entries.nextElement();
if (!entry.isDirectory()){
final String name = entry.getName();
final String ext = FilenameUtils.getExtension(name);
final InputStream in = new BufferedInputStream(archive.getInputStream(entry));
final File outFile = new File(outputDirectory, fileName!=null?new StringBuilder(fileName).append(".").append(ext).toString():name);
final OutputStream out = new BufferedOutputStream(new FileOutputStream(outFile));
IOUtils.copyStream(in, out, true, true);
}
else {
//if the entry is a directory attempt to make it
new File(outputDirectory, entry.getName()).mkdirs();
}
}
}
finally {
try {
archive.close();
}catch (Throwable e) {
if(LOGGER.isLoggable(Level.FINE))
LOGGER.isLoggable(Level.FINE);
}
}
}
/**
* Singleton
*/
private IOUtils() {
}
}