/*
* Copyright 2010-2013, the original author or authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudbees.clickstack.util;
import com.cloudbees.clickstack.util.exception.RuntimeIOException;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.*;
/**
* @author <a href="mailto:cleclerc@cloudbees.com">Cyrille Le Clerc</a>
*/
public class Files2 {
final static Set<PosixFilePermission> PERMISSION_R = Collections.unmodifiableSet(PosixFilePermissions.fromString("rw-r-----")); // grant 'w' to owner
final static Set<PosixFilePermission> PERMISSION_RX = Collections.unmodifiableSet(PosixFilePermissions.fromString("rwxr-x---")); // grant 'w' to owner
final static Set<PosixFilePermission> PERMISSION_RW = Collections.unmodifiableSet(PosixFilePermissions.fromString("rw-rw----"));
final static Set<PosixFilePermission> PERMISSION_RWX = Collections.unmodifiableSet(PosixFilePermissions.fromString("rwxrwx---"));
final static Set<PosixFilePermission> PERMISSION_750 = Collections.unmodifiableSet(PosixFilePermissions.fromString("rwxr-x---"));
final static Set<PosixFilePermission> PERMISSION_770 = Collections.unmodifiableSet(PosixFilePermissions.fromString("rwxrwx---"));
final static Set<PosixFilePermission> PERMISSION_640 = Collections.unmodifiableSet(PosixFilePermissions.fromString("rw-r-----"));
private static final Logger logger = LoggerFactory.getLogger(Files2.class);
/**
* Delete given {@code dir} and its content ({@code rm -rf}).
*
* @param dir
* @throws RuntimeIOException
*/
public static void deleteDirectory(@Nonnull Path dir) throws RuntimeIOException {
try {
Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file,
BasicFileAttributes attrs) throws IOException {
logger.trace("Delete file: {} ...", file);
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir,
IOException exc) throws IOException {
if (exc == null) {
logger.trace("Delete dir: {} ...", dir);
Files.delete(dir);
return FileVisitResult.CONTINUE;
} else {
throw exc;
}
}
});
} catch (IOException e) {
throw new RuntimeIOException("Exception deleting '" + dir + "'", e);
}
}
public static void chmodReadOnly(@Nonnull Path path) throws RuntimeIOException {
SimpleFileVisitor<Path> setReadOnlyFileVisitor = new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (Files.isDirectory(file)) {
throw new IllegalStateException("no dir expected here");
} else {
Files.setPosixFilePermissions(file, PERMISSION_R);
}
return super.visitFile(file, attrs);
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
Files.setPosixFilePermissions(dir, PERMISSION_RX);
return super.preVisitDirectory(dir, attrs);
}
};
try {
Files.walkFileTree(path, setReadOnlyFileVisitor);
} catch (IOException e) {
throw new RuntimeIOException("Exception changing permissions to readonly for " + path, e);
}
}
public static void chmodSetReadOnly(@Nonnull Path path) throws RuntimeIOException {
chmodOverwritePermissions(path, PERMISSION_R, PERMISSION_RX);
}
public static void chmodAddReadExecute(@Nonnull Path path) throws RuntimeIOException {
chmodAddPermissions(path, PERMISSION_RX, PERMISSION_RX);
}
public static void chmodAddReadWrite(@Nonnull Path path) throws RuntimeIOException {
chmodAddPermissions(path, PERMISSION_RW, PERMISSION_RWX);
}
/**
* @deprecated use {@link #chmodAddReadExecute(java.nio.file.Path)}
*/
@Deprecated
public static void chmodReadExecute(Path path) throws RuntimeIOException {
chmodAddReadExecute(path);
}
/**
* @deprecated use {@link #chmodAddReadWrite(java.nio.file.Path)}
*/
@Deprecated
public static void chmodReadWrite(Path path) throws RuntimeIOException {
chmodAddReadWrite(path);
}
/**
* Returns a zip file system
*
* @param zipFile to construct the file system from
* @param create true if the zip file should be created
* @return a zip file system
* @throws java.io.IOException
*/
private static FileSystem createZipFileSystem(Path zipFile, boolean create) throws IOException {
// convert the filename to a URI
final URI uri = URI.create("jar:file:" + zipFile.toUri().getPath());
final Map<String, String> env = new HashMap<>();
if (create) {
env.put("create", "true");
}
return FileSystems.newFileSystem(uri, env);
}
/**
* Unzips the specified zip file to the specified destination directory.
* Replaces any files in the destination, if they already exist.
*
* @param zipFilename the name of the zip file to extract
* @param destDirname the directory to unzip to
* @throws RuntimeIOException
*/
public static void unzip(String zipFilename, String destDirname) throws RuntimeIOException {
Path zipFile = Paths.get(zipFilename);
Path destDir = Paths.get(destDirname);
unzip(zipFile, destDir);
}
/**
* Copy every file of given {@code zipFile} beginning with given {@code zipSubPath} to {@code destDir}
*
* @param zipFile
* @param zipSubPath
* @param destDir
* @throws RuntimeIOException
*/
@Nullable
public static Path unzipSubFileIfExists(@Nonnull Path zipFile, @Nonnull String zipSubPath, @Nonnull final Path destDir) throws RuntimeIOException {
try {
//if the destination doesn't exist, create it
if (Files.notExists(destDir)) {
logger.trace("Create dir: {}", destDir);
Files.createDirectories(destDir);
}
try (FileSystem zipFileSystem = createZipFileSystem(zipFile, false)) {
final Path root = zipFileSystem.getPath("/");
Path subFile = root.resolve(zipSubPath);
if (Files.exists(subFile)) {
// make file path relative
Path destFile;
if (Strings2.beginWith(zipSubPath, "/")) {
destFile = destDir.resolve("." + zipSubPath);
} else {
destFile = destDir.resolve(zipSubPath);
}
// create parent dirs if needed
Files.createDirectories(destFile.getParent());
// copy
return Files.copy(subFile, destFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
} else {
return null;
}
}
} catch (IOException e) {
throw new RuntimeIOException("Exception expanding " + zipFile + ":" + zipSubPath + " to " + destDir, e);
}
}
/**
* Copy every file of given {@code zipFile} beginning with given {@code zipSubPath} to {@code destDir}
*
* @param zipFile
* @param zipSubPath
* @param destDir
* @throws RuntimeIOException
*/
public static void unzipSubDirectoryIfExists(@Nonnull Path zipFile, @Nonnull String zipSubPath, @Nonnull final Path destDir) throws RuntimeIOException {
try {
//if the destination doesn't exist, create it
if (Files.notExists(destDir)) {
logger.trace("Create dir: {}", destDir);
Files.createDirectories(destDir);
}
try (FileSystem zipFileSystem = createZipFileSystem(zipFile, false)) {
final Path root = zipFileSystem.getPath(zipSubPath);
if (Files.notExists(root)) {
logger.trace("Zip sub path {} does not exist in {}", zipSubPath, zipFile);
return;
}
//walk the zip file tree and copy files to the destination
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file,
BasicFileAttributes attrs) throws IOException {
final Path destFile = Paths.get(destDir.toString(), root.relativize(file).toString());
logger.trace("Extract file {} to {} as {}", file, destDir, destFile);
Files.copy(file, destFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) throws IOException {
dir.relativize(root).toString();
final Path dirToCreate = Paths.get(destDir.toString(), root.relativize(dir).toString());
if (Files.notExists(dirToCreate)) {
logger.trace("Create dir {}", dirToCreate);
Files.createDirectory(dirToCreate);
}
return FileVisitResult.CONTINUE;
}
});
}
} catch (IOException e) {
throw new RuntimeIOException("Exception expanding " + zipFile + ":" + zipSubPath + " to " + destDir, e);
}
}
public static void unzip(@Nonnull Path zipFile, @Nonnull final Path destDir) throws RuntimeIOException {
try {
//if the destination doesn't exist, create it
if (Files.notExists(destDir)) {
logger.trace("Create dir: {}", destDir);
Files.createDirectories(destDir);
}
try (FileSystem zipFileSystem = createZipFileSystem(zipFile, false)) {
final Path root = zipFileSystem.getPath("/");
//walk the zip file tree and copy files to the destination
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file,
BasicFileAttributes attrs) throws IOException {
final Path destFile = Paths.get(destDir.toString(), file.toString());
logger.trace("Extract file {} to {}", file, destDir);
try {
Files.copy(file, destFile, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
logger.warn("Exception copying file '" + file + "' to '" + destFile + "', ignore file", e);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) throws IOException {
final Path dirToCreate = Paths.get(destDir.toString(), dir.toString());
if (Files.notExists(dirToCreate)) {
logger.trace("Create dir {}", dirToCreate);
try {
Files.createDirectory(dirToCreate);
} catch (IOException e) {
logger.warn("Exception creating directory '" + dirToCreate + "' for '" + dir + "', ignore dir subtree", e);
return FileVisitResult.SKIP_SUBTREE;
}
}
return FileVisitResult.CONTINUE;
}
});
}
} catch (IOException e) {
throw new RuntimeIOException("Exception expanding " + zipFile + " to " + destDir, e);
}
}
/**
* Uncompress the specified tgz file to the specified destination directory.
* Replaces any files in the destination, if they already exist.
*
* @param tgzFilename the name of the zip file to extract
* @param destDirname the directory to unzip to
* @throws RuntimeIOException
*/
public static void untgz(String tgzFilename, String destDirname) throws RuntimeIOException {
Path tgzFile = Paths.get(tgzFilename);
Path destDir = Paths.get(destDirname);
untgz(tgzFile, destDir);
}
/**
* TODO recopy file permissions
* <p/>
* Uncompress the specified tgz file to the specified destination directory.
* Replaces any files in the destination, if they already exist.
*
* @param tgzFile the name of the zip file to extract
* @param destDir the directory to unzip to
* @throws RuntimeIOException
*/
public static void untgz(@Nonnull Path tgzFile, @Nonnull Path destDir) throws RuntimeIOException {
try {
//if the destination doesn't exist, create it
if (Files.notExists(destDir)) {
logger.trace("Create dir: {}", destDir);
Files.createDirectories(destDir);
}
TarArchiveInputStream in = new TarArchiveInputStream(new GzipCompressorInputStream(Files.newInputStream(tgzFile)));
TarArchiveEntry entry;
while ((entry = in.getNextTarEntry()) != null) {
if (entry.isDirectory()) {
Path dir = destDir.resolve(entry.getName());
logger.trace("Create dir {}", dir);
Files.createDirectories(dir);
} else {
Path file = destDir.resolve(entry.getName());
logger.trace("Create file {}: {} bytes", file, entry.getSize());
OutputStream out = Files.newOutputStream(file);
IOUtils.copy(in, out);
out.close();
}
}
in.close();
} catch (IOException e) {
throw new RuntimeIOException("Exception expanding " + tgzFile + " to " + destDir, e);
}
}
/**
* For debugging purpose. Dump the tree view of the dir in {@code stderr}
*
* @param path
* @throws IOException
*/
public static void dump(@Nonnull Path path) throws RuntimeIOException {
System.err.println("## DUMP FOLDER TREE ##");
dump(path, 0);
}
private static void dump(@Nonnull Path path, int depth) throws RuntimeIOException {
try {
depth++;
String icon = Files.isDirectory(path) ? " + " : " |- ";
System.out.println(Strings.repeat(" ", depth) + icon + path.getFileName() + "\t" + PosixFilePermissions.toString(Files.getPosixFilePermissions(path)));
if (Files.isDirectory(path)) {
DirectoryStream<Path> children = Files.newDirectoryStream(path);
for (Path child : children) {
dump(child, depth);
}
}
} catch (IOException e) {
throw new RuntimeIOException("Exception dumping " + path, e);
}
}
/**
* Copy content for {@code srcDir} to {@code destDir}
*
* @param srcDir
* @param destDir
* @throws RuntimeIOException
*/
public static void copyDirectoryContent(@Nonnull final Path srcDir, @Nonnull final Path destDir) throws RuntimeIOException {
logger.trace("Copy from {} to {}", srcDir, destDir);
FileVisitor<Path> copyDirVisitor = new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
Path targetPath = destDir.resolve(srcDir.relativize(dir));
if (!Files.exists(targetPath)) {
Files.createDirectory(targetPath);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.copy(file, destDir.resolve(srcDir.relativize(file)), StandardCopyOption.REPLACE_EXISTING);
return FileVisitResult.CONTINUE;
}
};
try {
Files.walkFileTree(srcDir, copyDirVisitor);
} catch (IOException e) {
throw new RuntimeIOException("Exception copying content of dir " + srcDir + " to " + destDir, e);
}
}
/**
* Copy given {@code srcFile} to given {@code destDir}.
*
* @param srcFile
* @param destDir destination directory, must exist
* @return
* @throws RuntimeIOException
*/
@Nonnull
public static Path copyToDirectory(@Nonnull Path srcFile, @Nonnull Path destDir) throws RuntimeIOException {
Preconditions.checkArgument(Files.exists(srcFile), "Src %s not found");
Preconditions.checkArgument(!Files.isDirectory(srcFile), "Src %s is a directory");
Preconditions.checkArgument(Files.exists(destDir), "Dest %s not found");
Preconditions.checkArgument(Files.isDirectory(destDir), "Dest %s is not a directory");
try {
return Files.copy(srcFile, destDir.resolve(srcFile.getFileName()));
} catch (IOException e) {
throw new RuntimeIOException("Exception copying " + srcFile.getFileName() + " to " + srcFile, e);
}
}
@Nonnull
public static Path copyArtifactToDirectory(@Nonnull Path sourceDir, @Nonnull String artifactId, @Nonnull Path dest) throws RuntimeIOException {
Path source = findArtifact(sourceDir, artifactId);
try {
return Files.copy(source, dest.resolve(source.getFileName()));
} catch (IOException e) {
throw new RuntimeIOException("Exception copying " + source.getFileName() + " to " + sourceDir, e);
}
}
/**
* Find jar file with name beginning with given {@code artifactId} in given {@code srcDir}.
*
* @param srcDir
* @param artifactId
* @return
* @throws RuntimeIOException
* @see #findArtifact(java.nio.file.Path, String, String)
*/
@Nonnull
public static Path findArtifact(@Nonnull Path srcDir, @Nonnull String artifactId) throws RuntimeIOException, IllegalStateException {
return findArtifact(srcDir, artifactId, "jar");
}
/**
* @deprecated use {@link #findUniqueDirectoryBeginningWith(java.nio.file.Path, String)}
*/
@Deprecated
@Nonnull
public static Path findUniqueFolderBeginningWith(@Nonnull Path source, @Nullable final String pattern) throws RuntimeIOException, IllegalStateException {
return findUniqueDirectoryBeginningWith(source, pattern);
}
/**
* @param srcDir
* @param pattern
* @return
* @throws RuntimeIOException
* @throws IllegalStateException More or less than 1 child dir found
*/
@Nonnull
public static Path findUniqueDirectoryBeginningWith(@Nonnull Path srcDir, @Nonnull final String pattern) throws RuntimeIOException, IllegalStateException {
Preconditions.checkArgument(Files.isDirectory(srcDir), "Source %s is not a directory", srcDir.toAbsolutePath());
DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {
@Override
public boolean accept(Path entry) throws IOException {
String fileName = entry.getFileName().toString();
if (pattern == null) {
return true;
} else if (fileName.startsWith(pattern)) {
return true;
} else {
return false;
}
}
};
try (DirectoryStream<Path> paths = Files.newDirectoryStream(srcDir, filter)) {
try {
return Iterables.getOnlyElement(paths);
} catch (NoSuchElementException e) {
throw new IllegalStateException("Directory beginning with '" + pattern + "' not found in path: " + srcDir + ", absolutePath: " + srcDir.toAbsolutePath());
} catch (IllegalArgumentException e) {
throw new IllegalStateException("More than 1 directory beginning with '" + pattern + "' found in path: " + srcDir + ", absolutePath: " + srcDir.toAbsolutePath() + " -> " + paths);
}
} catch (IOException e) {
throw new RuntimeIOException("Exception finding unique child directory beginning with " + filter + "in " + srcDir);
}
}
/**
* @param srcDir
* @return
* @throws RuntimeIOException
* @throws IllegalStateException More or less than 1 child dir found
*/
@Nonnull
public static Path findUniqueChildDirectory(@Nonnull Path srcDir) throws RuntimeIOException, IllegalStateException {
Preconditions.checkArgument(Files.isDirectory(srcDir), "Source %s is not a directory", srcDir.toAbsolutePath());
try (DirectoryStream<Path> paths = Files.newDirectoryStream(srcDir)) {
try {
return Iterables.getOnlyElement(paths);
} catch (NoSuchElementException e) {
throw new IllegalStateException("No child directory found in : " + srcDir + ", absolutePath: " + srcDir.toAbsolutePath());
} catch (IllegalArgumentException e) {
throw new IllegalStateException("More than 1 child directory found in path: " + srcDir + ", absolutePath: " + srcDir.toAbsolutePath() + " -> " + paths);
}
} catch (IOException e) {
throw new RuntimeIOException("Exception finding unique child directory in " + srcDir);
}
}
/**
* Find a file matching {@code $artifactId*$type} in the given {@code srcDir}.
*
* @param srcDir
* @param artifactId
* @param type
* @return
* @throws IllegalStateException More or less than 1 matching artifact found
* @throws RuntimeIOException
*/
@Nonnull
public static Path findArtifact(@Nonnull Path srcDir, @Nonnull final String artifactId, @Nonnull final String type) throws RuntimeIOException, IllegalStateException {
Preconditions.checkArgument(Files.isDirectory(srcDir), "Source %s is not a directory", srcDir.toAbsolutePath());
DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {
@Override
public boolean accept(Path entry) throws IOException {
String fileName = entry.getFileName().toString();
if (fileName.startsWith(artifactId) && fileName.endsWith("." + type)) {
return true;
} else {
return false;
}
}
};
try (DirectoryStream<Path> paths = Files.newDirectoryStream(srcDir, filter)) {
try {
return Iterables.getOnlyElement(paths);
} catch (NoSuchElementException e) {
throw new IllegalStateException("Artifact '" + artifactId + ":" + type + "' not found in path: " + srcDir + ", absolutePath: " + srcDir.toAbsolutePath());
} catch (IllegalArgumentException e) {
throw new IllegalStateException("More than 1 version of artifact '" + artifactId + ":" + type + "' found in path: " + srcDir + ", absolutePath: " + srcDir.toAbsolutePath() + " -> " + paths);
}
} catch (IOException e) {
throw new RuntimeIOException("Exception finding artifact " + artifactId + "@" + type + " in " + srcDir);
}
}
/**
* Update file and dir permissions.
*
* @param path
* @param filePermissions
* @param dirPermissions
* @throws RuntimeIOException
*/
private static void chmodOverwritePermissions(@Nonnull Path path, @Nonnull final Set<PosixFilePermission> filePermissions, @Nonnull final Set<PosixFilePermission> dirPermissions) throws RuntimeIOException {
if (!Files.exists(path)) {
throw new IllegalArgumentException("Given path " + path + " does not exist");
}
SimpleFileVisitor<Path> setReadOnlyFileVisitor = new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (Files.isDirectory(file)) {
throw new IllegalStateException("No dir expected here: " + file);
} else {
Files.setPosixFilePermissions(file, filePermissions);
}
return super.visitFile(file, attrs);
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
Files.setPosixFilePermissions(dir, dirPermissions);
return super.preVisitDirectory(dir, attrs);
}
};
try {
Files.walkFileTree(path, setReadOnlyFileVisitor);
} catch (IOException e) {
throw new RuntimeIOException("Exception setting permissions file permissions to " + filePermissions + " and folder permissions to " + dirPermissions + " on " + path, e);
}
}
/**
* Update file and dir permissions.
*
* @param path
* @param filePermissions
* @param dirPermissions
* @throws RuntimeIOException
*/
private static void chmodAddPermissions(@Nonnull Path path, @Nonnull final Set<PosixFilePermission> filePermissions, @Nonnull final Set<PosixFilePermission> dirPermissions) throws RuntimeIOException {
if (!Files.exists(path)) {
throw new IllegalArgumentException("Given path " + path + " does not exist");
}
SimpleFileVisitor<Path> setReadOnlyFileVisitor = new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (Files.isDirectory(file)) {
throw new IllegalStateException("No dir expected here: " + file);
} else {
Set<PosixFilePermission> existingPermissions = Files.getPosixFilePermissions(file);
Files.setPosixFilePermissions(file, Sets.union(existingPermissions, filePermissions));
}
return super.visitFile(file, attrs);
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
Set<PosixFilePermission> existingPermissions = Files.getPosixFilePermissions(dir);
Files.setPosixFilePermissions(dir, Sets.union(existingPermissions, dirPermissions));
return super.preVisitDirectory(dir, attrs);
}
};
try {
Files.walkFileTree(path, setReadOnlyFileVisitor);
} catch (IOException e) {
throw new RuntimeIOException("Exception setting permissions file permissions to " + filePermissions + " and folder permissions to " + dirPermissions + " on " + path, e);
}
}
}