/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.StringReader;
import java.util.Arrays;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
/**
* Class for creating hardlinks.
* Supports Unix/Linux, Windows via winutils , and Mac OS X.
*
* The HardLink class was formerly a static inner class of FSUtil,
* and the methods provided were blatantly non-thread-safe.
* To enable volume-parallel Update snapshots, we now provide static
* threadsafe methods that allocate new buffer string arrays
* upon each call. We also provide an API to hardlink all files in a
* directory with a single command, which is up to 128 times more
* efficient - and minimizes the impact of the extra buffer creations.
*/
public class HardLink {
private static HardLinkCommandGetter getHardLinkCommand;
public final LinkStats linkStats; //not static
//initialize the command "getters" statically, so can use their
//methods without instantiating the HardLink object
static {
if (Shell.WINDOWS) {
// Windows
getHardLinkCommand = new HardLinkCGWin();
} else {
// Unix or Linux
getHardLinkCommand = new HardLinkCGUnix();
//override getLinkCountCommand for the particular Unix variant
//Linux is already set as the default - {"stat","-c%h", null}
if (Shell.MAC || Shell.FREEBSD) {
String[] linkCountCmdTemplate = {"/usr/bin/stat","-f%l", null};
HardLinkCGUnix.setLinkCountCmdTemplate(linkCountCmdTemplate);
} else if (Shell.SOLARIS) {
String[] linkCountCmdTemplate = {"ls","-l", null};
HardLinkCGUnix.setLinkCountCmdTemplate(linkCountCmdTemplate);
}
}
}
public HardLink() {
linkStats = new LinkStats();
}
/**
* This abstract class bridges the OS-dependent implementations of the
* needed functionality for creating hardlinks and querying link counts.
* The particular implementation class is chosen during
* static initialization phase of the HardLink class.
* The "getter" methods construct shell command strings for various purposes.
*/
private static abstract class HardLinkCommandGetter {
/**
* Get the command string needed to hardlink a bunch of files from
* a single source directory into a target directory. The source directory
* is not specified here, but the command will be executed using the source
* directory as the "current working directory" of the shell invocation.
*
* @param fileBaseNames - array of path-less file names, relative
* to the source directory
* @param linkDir - target directory where the hardlinks will be put
* @return - an array of Strings suitable for use as a single shell command
* @throws IOException - if any of the file or path names misbehave
*/
abstract String[] linkMult(String[] fileBaseNames, File linkDir)
throws IOException;
/**
* Get the command string needed to hardlink a single file
*/
abstract String[] linkOne(File file, File linkName) throws IOException;
/**
* Get the command string to query the hardlink count of a file
*/
abstract String[] linkCount(File file) throws IOException;
/**
* Calculate the total string length of the shell command
* resulting from execution of linkMult, plus the length of the
* source directory name (which will also be provided to the shell)
*
* @param fileDir - source directory, parent of fileBaseNames
* @param fileBaseNames - array of path-less file names, relative
* to the source directory
* @param linkDir - target directory where the hardlinks will be put
* @return - total data length (must not exceed maxAllowedCmdArgLength)
* @throws IOException
*/
abstract int getLinkMultArgLength(
File fileDir, String[] fileBaseNames, File linkDir)
throws IOException;
/**
* Get the maximum allowed string length of a shell command on this OS,
* which is just the documented minimum guaranteed supported command
* length - aprx. 32KB for Unix, and 8KB for Windows.
*/
abstract int getMaxAllowedCmdArgLength();
}
/**
* Implementation of HardLinkCommandGetter class for Unix
*/
static class HardLinkCGUnix extends HardLinkCommandGetter {
private static String[] hardLinkCommand = {"ln", null, null};
private static String[] hardLinkMultPrefix = {"ln"};
private static String[] hardLinkMultSuffix = {null};
private static String[] getLinkCountCommand = {"stat","-c%h", null};
//Unix guarantees at least 32K bytes cmd length.
//Subtract another 64b to allow for Java 'exec' overhead
private static final int maxAllowedCmdArgLength = 32*1024 - 65;
private static synchronized
void setLinkCountCmdTemplate(String[] template) {
//May update this for specific unix variants,
//after static initialization phase
getLinkCountCommand = template;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkOne(java.io.File, java.io.File)
*/
@Override
String[] linkOne(File file, File linkName)
throws IOException {
String[] buf = new String[hardLinkCommand.length];
System.arraycopy(hardLinkCommand, 0, buf, 0, hardLinkCommand.length);
//unix wants argument order: "ln <existing> <new>"
buf[1] = FileUtil.makeShellPath(file, true);
buf[2] = FileUtil.makeShellPath(linkName, true);
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkMult(java.lang.String[], java.io.File)
*/
@Override
String[] linkMult(String[] fileBaseNames, File linkDir)
throws IOException {
String[] buf = new String[fileBaseNames.length
+ hardLinkMultPrefix.length
+ hardLinkMultSuffix.length];
int mark=0;
System.arraycopy(hardLinkMultPrefix, 0, buf, mark,
hardLinkMultPrefix.length);
mark += hardLinkMultPrefix.length;
System.arraycopy(fileBaseNames, 0, buf, mark, fileBaseNames.length);
mark += fileBaseNames.length;
buf[mark] = FileUtil.makeShellPath(linkDir, true);
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkCount(java.io.File)
*/
@Override
String[] linkCount(File file)
throws IOException {
String[] buf = new String[getLinkCountCommand.length];
System.arraycopy(getLinkCountCommand, 0, buf, 0,
getLinkCountCommand.length);
buf[getLinkCountCommand.length - 1] = FileUtil.makeShellPath(file, true);
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getLinkMultArgLength(java.io.File, java.lang.String[], java.io.File)
*/
@Override
int getLinkMultArgLength(File fileDir, String[] fileBaseNames, File linkDir)
throws IOException{
int sum = 0;
for (String x : fileBaseNames) {
// add 1 to account for terminal null or delimiter space
sum += 1 + ((x == null) ? 0 : x.length());
}
sum += 2 + FileUtil.makeShellPath(fileDir, true).length()
+ FileUtil.makeShellPath(linkDir, true).length();
//add the fixed overhead of the hardLinkMult prefix and suffix
sum += 3; //length("ln") + 1
return sum;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getMaxAllowedCmdArgLength()
*/
@Override
int getMaxAllowedCmdArgLength() {
return maxAllowedCmdArgLength;
}
}
/**
* Implementation of HardLinkCommandGetter class for Windows
*/
static class HardLinkCGWin extends HardLinkCommandGetter {
//The Windows command getter impl class and its member fields are
//package-private ("default") access instead of "private" to assist
//unit testing (sort of) on non-Win servers
static String CMD_EXE = "cmd.exe";
static String[] hardLinkCommand = {
Shell.WINUTILS,"hardlink","create", null, null};
static String[] hardLinkMultPrefix = {
CMD_EXE, "/q", "/c", "for", "%f", "in", "("};
static String hardLinkMultDir = "\\%f";
static String[] hardLinkMultSuffix = {
")", "do", Shell.WINUTILS, "hardlink", "create", null,
"%f"};
static String[] getLinkCountCommand = {
Shell.WINUTILS, "hardlink", "stat", null};
//Windows guarantees only 8K - 1 bytes cmd length.
//Subtract another 64b to allow for Java 'exec' overhead
static final int maxAllowedCmdArgLength = 8*1024 - 65;
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkOne(java.io.File, java.io.File)
*/
@Override
String[] linkOne(File file, File linkName)
throws IOException {
String[] buf = new String[hardLinkCommand.length];
System.arraycopy(hardLinkCommand, 0, buf, 0, hardLinkCommand.length);
//windows wants argument order: "create <new> <existing>"
buf[4] = file.getCanonicalPath();
buf[3] = linkName.getCanonicalPath();
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkMult(java.lang.String[], java.io.File)
*/
@Override
String[] linkMult(String[] fileBaseNames, File linkDir)
throws IOException {
String[] buf = new String[fileBaseNames.length
+ hardLinkMultPrefix.length
+ hardLinkMultSuffix.length];
String td = linkDir.getCanonicalPath() + hardLinkMultDir;
int mark=0;
System.arraycopy(hardLinkMultPrefix, 0, buf, mark,
hardLinkMultPrefix.length);
mark += hardLinkMultPrefix.length;
System.arraycopy(fileBaseNames, 0, buf, mark, fileBaseNames.length);
mark += fileBaseNames.length;
System.arraycopy(hardLinkMultSuffix, 0, buf, mark,
hardLinkMultSuffix.length);
mark += hardLinkMultSuffix.length;
buf[mark - 2] = td;
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkCount(java.io.File)
*/
@Override
String[] linkCount(File file)
throws IOException {
String[] buf = new String[getLinkCountCommand.length];
System.arraycopy(getLinkCountCommand, 0, buf, 0,
getLinkCountCommand.length);
buf[getLinkCountCommand.length - 1] = file.getCanonicalPath();
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getLinkMultArgLength(java.io.File, java.lang.String[], java.io.File)
*/
@Override
int getLinkMultArgLength(File fileDir, String[] fileBaseNames, File linkDir)
throws IOException {
int sum = 0;
for (String x : fileBaseNames) {
// add 1 to account for terminal null or delimiter space
sum += 1 + ((x == null) ? 0 : x.length());
}
sum += 2 + fileDir.getCanonicalPath().length() +
linkDir.getCanonicalPath().length();
//add the fixed overhead of the hardLinkMult command
//(prefix, suffix, and Dir suffix)
sum += (CMD_EXE + " /q /c for %f in ( ) do "
+ Shell.WINUTILS + " hardlink create \\%f %f").length();
return sum;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getMaxAllowedCmdArgLength()
*/
@Override
int getMaxAllowedCmdArgLength() {
return maxAllowedCmdArgLength;
}
}
/**
* Calculate the nominal length of all contributors to the total
* commandstring length, including fixed overhead of the OS-dependent
* command. It's protected rather than private, to assist unit testing,
* but real clients are not expected to need it -- see the way
* createHardLinkMult() uses it internally so the user doesn't need to worry
* about it.
*
* @param fileDir - source directory, parent of fileBaseNames
* @param fileBaseNames - array of path-less file names, relative
* to the source directory
* @param linkDir - target directory where the hardlinks will be put
* @return - total data length (must not exceed maxAllowedCmdArgLength)
* @throws IOException
*/
protected static int getLinkMultArgLength(
File fileDir, String[] fileBaseNames, File linkDir)
throws IOException {
return getHardLinkCommand.getLinkMultArgLength(fileDir,
fileBaseNames, linkDir);
}
/**
* Return this private value for use by unit tests.
* Shell commands are not allowed to have a total string length
* exceeding this size.
*/
protected static int getMaxAllowedCmdArgLength() {
return getHardLinkCommand.getMaxAllowedCmdArgLength();
}
/*
* ****************************************************
* Complexity is above. User-visible functionality is below
* ****************************************************
*/
/**
* Creates a hardlink
* @param file - existing source file
* @param linkName - desired target link file
*/
public static void createHardLink(File file, File linkName)
throws IOException {
if (file == null) {
throw new IOException(
"invalid arguments to createHardLink: source file is null");
}
if (linkName == null) {
throw new IOException(
"invalid arguments to createHardLink: link name is null");
}
// construct and execute shell command
String[] hardLinkCommand = getHardLinkCommand.linkOne(file, linkName);
ShellCommandExecutor shexec = new ShellCommandExecutor(hardLinkCommand);
try {
shexec.execute();
} catch (ExitCodeException e) {
throw new IOException("Failed to execute command " +
Arrays.toString(hardLinkCommand) +
"; command output: \"" + shexec.getOutput() + "\"" +
"; WrappedException: \"" + e.getMessage() + "\"");
}
}
/**
* Creates hardlinks from multiple existing files within one parent
* directory, into one target directory.
* @param parentDir - directory containing source files
* @param fileBaseNames - list of path-less file names, as returned by
* parentDir.list()
* @param linkDir - where the hardlinks should be put. It must already exist.
*
* If the list of files is too long (overflows maxAllowedCmdArgLength),
* we will automatically split it into multiple invocations of the
* underlying method.
*/
public static void createHardLinkMult(File parentDir, String[] fileBaseNames,
File linkDir) throws IOException {
//This is the public method all non-test clients are expected to use.
//Normal case - allow up to maxAllowedCmdArgLength characters in the cmd
createHardLinkMult(parentDir, fileBaseNames, linkDir,
getHardLinkCommand.getMaxAllowedCmdArgLength());
}
/*
* Implements {@link createHardLinkMult} with added variable "maxLength",
* to ease unit testing of the auto-splitting feature for long lists.
* Likewise why it returns "callCount", the number of sub-arrays that
* the file list had to be split into.
* Non-test clients are expected to call the public method instead.
*/
protected static int createHardLinkMult(File parentDir,
String[] fileBaseNames, File linkDir, int maxLength)
throws IOException {
if (parentDir == null) {
throw new IOException(
"invalid arguments to createHardLinkMult: parent directory is null");
}
if (linkDir == null) {
throw new IOException(
"invalid arguments to createHardLinkMult: link directory is null");
}
if (fileBaseNames == null) {
throw new IOException(
"invalid arguments to createHardLinkMult: "
+ "filename list can be empty but not null");
}
if (fileBaseNames.length == 0) {
//the OS cmds can't handle empty list of filenames,
//but it's legal, so just return.
return 0;
}
if (!linkDir.exists()) {
throw new FileNotFoundException(linkDir + " not found.");
}
//if the list is too long, split into multiple invocations
int callCount = 0;
if (getLinkMultArgLength(parentDir, fileBaseNames, linkDir) > maxLength
&& fileBaseNames.length > 1) {
String[] list1 = Arrays.copyOf(fileBaseNames, fileBaseNames.length/2);
callCount += createHardLinkMult(parentDir, list1, linkDir, maxLength);
String[] list2 = Arrays.copyOfRange(fileBaseNames, fileBaseNames.length/2,
fileBaseNames.length);
callCount += createHardLinkMult(parentDir, list2, linkDir, maxLength);
return callCount;
} else {
callCount = 1;
}
// construct and execute shell command
String[] hardLinkCommand = getHardLinkCommand.linkMult(fileBaseNames,
linkDir);
ShellCommandExecutor shexec = new ShellCommandExecutor(hardLinkCommand,
parentDir, null, 0L);
try {
shexec.execute();
} catch (ExitCodeException e) {
throw new IOException(shexec.getOutput() + e.getMessage());
}
return callCount;
}
/**
* Retrieves the number of links to the specified file.
*/
public static int getLinkCount(File fileName) throws IOException {
if (fileName == null) {
throw new IOException(
"invalid argument to getLinkCount: file name is null");
}
if (!fileName.exists()) {
throw new FileNotFoundException(fileName + " not found.");
}
// construct and execute shell command
String[] cmd = getHardLinkCommand.linkCount(fileName);
String inpMsg = null;
String errMsg = null;
int exitValue = -1;
BufferedReader in = null;
ShellCommandExecutor shexec = new ShellCommandExecutor(cmd);
try {
shexec.execute();
in = new BufferedReader(new StringReader(shexec.getOutput()));
inpMsg = in.readLine();
exitValue = shexec.getExitCode();
if (inpMsg == null || exitValue != 0) {
throw createIOException(fileName, inpMsg, errMsg, exitValue, null);
}
if (Shell.SOLARIS) {
String[] result = inpMsg.split("\\s+");
return Integer.parseInt(result[1]);
} else {
return Integer.parseInt(inpMsg);
}
} catch (ExitCodeException e) {
inpMsg = shexec.getOutput();
errMsg = e.getMessage();
exitValue = e.getExitCode();
throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
} catch (NumberFormatException e) {
throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
} finally {
IOUtils.closeStream(in);
}
}
/* Create an IOException for failing to get link count. */
private static IOException createIOException(File f, String message,
String error, int exitvalue, Exception cause) {
final String s = "Failed to get link count on file " + f
+ ": message=" + message
+ "; error=" + error
+ "; exit value=" + exitvalue;
return (cause == null) ? new IOException(s) : new IOException(s, cause);
}
/**
* HardLink statistics counters and methods.
* Not multi-thread safe, obviously.
* Init is called during HardLink instantiation, above.
*
* These are intended for use by knowledgeable clients, not internally,
* because many of the internal methods are static and can't update these
* per-instance counters.
*/
public static class LinkStats {
public int countDirs = 0;
public int countSingleLinks = 0;
public int countMultLinks = 0;
public int countFilesMultLinks = 0;
public int countEmptyDirs = 0;
public int countPhysicalFileCopies = 0;
public void clear() {
countDirs = 0;
countSingleLinks = 0;
countMultLinks = 0;
countFilesMultLinks = 0;
countEmptyDirs = 0;
countPhysicalFileCopies = 0;
}
public String report() {
return "HardLinkStats: " + countDirs + " Directories, including "
+ countEmptyDirs + " Empty Directories, "
+ countSingleLinks
+ " single Link operations, " + countMultLinks
+ " multi-Link operations, linking " + countFilesMultLinks
+ " files, total " + (countSingleLinks + countFilesMultLinks)
+ " linkable files. Also physically copied "
+ countPhysicalFileCopies + " other files.";
}
}
}