/* The contents of this file are subject to the terms
* of the Common Development and Distribution License
* (the License). You may not use this file except in
* compliance with the License.
*
* You can obtain a copy of the License at
* http://www.sun.com/cddl/cddl.html or
* install_dir/legal/LICENSE
* See the License for the specific language governing
* permission and limitations under the License.
*
* When distributing Covered Code, include this CDDL
* Header Notice in each file and include the License file
* at install_dir/legal/LICENSE.
* If applicable, add the following below the CDDL Header,
* with the fields enclosed by brackets [] replaced by
* your own identifying information:
* "Portions Copyrighted [year] [name of copyright owner]"
*
* $Id$
*
* Copyright 2005-2009 Sun Microsystems Inc. All Rights Reserved
*/
package com.sun.faban.harness.engine;
import com.sun.faban.common.*;
import com.sun.faban.harness.FabanHostUnknownException;
import com.sun.faban.harness.RemoteCallable;
import com.sun.faban.harness.ParamRepository;
import com.sun.faban.harness.ConfigurationException;
import com.sun.faban.harness.agent.CmdAgent;
import com.sun.faban.harness.agent.FileAgent;
import com.sun.faban.harness.agent.FileService;
import com.sun.faban.harness.common.Config;
import com.sun.faban.harness.common.HostRoles;
import com.sun.faban.harness.util.CmdMap;
import com.sun.faban.harness.util.FileHelper;
import com.sun.faban.harness.util.InterfaceProbe;
import java.io.*;
import java.net.*;
import java.rmi.RemoteException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* This file contains the class that implements the Command service API.
* The Command Service object is created by the Engine at the start of
* a run and it starts up the CmdAgent applications on all the
* machines and connects to them via RMI. In the API implementation,
* it identifies the particular CmdAgent and passes the call along.
*
* The CmdAgents take care of any error messages generated by the
* command and automatically log them to the run's error log.
* The CmdAgent's path will include the default faban bin
* directories (in addition to /usr/bin, /usr/sbin, /usr/ucb), so
* any faban executables will be found. Commands in any other
* path should be invoked with the full pathname of the command.
* The CmdAgent's environment will also include CLASSPATH set to
* the faban lib directory to find any Java classes.
*
* Shell commands or any commands whose output must be re-directed
* or piped (basically using shell) should be executed using syntax
* such as :
* "sh -c <command> [<args>] [> out]".
* IMPORTANT: There should be a single CmdService object in the
* entire framework or else multiple copies of the CmdAgent
* application will be spawned on the target machines.
* Therefore, this class is implemented as a Singleton.
* No public constructors are defined and the object cannot be cloned.
*
* @author Ramesh Ramachandran
* @see com.sun.faban.harness.agent.CmdAgent
* @see com.sun.faban.harness.engine.GenericBenchmark
*/
final public class CmdService { // The final keyword prevents clones
/** Sequential flag in FG mode. */
public static final int SEQUENTIAL = 1;
/** Parallel flag in FG mode. */
public static final int PARALLEL = 2;
private static Logger logger = Logger.getLogger(CmdService.class.getName());
private static CmdService cmds;
private ArrayList<CmdAgent> cmdp = new ArrayList<CmdAgent>();
private ArrayList<FileAgent> filep = new ArrayList<FileAgent>();
/** List of all machines. */
private ArrayList<String> machinesList = new ArrayList<String>();
private Properties hostInterfaces = new Properties();
private Registry registry;
private String master; // Name of faban master machine
private String masterAddress; // ip of faban master machine
private CommandHandle registryCmd;
private String javaHome;
private List<String> jvmOptions;
private HashMap<String, List<String>> binMap =
new HashMap<String, List<String>>();
private Map<String, String> ifMap;
private List<String> rsh, agent;
private HostRoles hostRoles;
CmdService() {
try {
master = (InetAddress.getLocalHost()).getHostName();
masterAddress = (InetAddress.getLocalHost()).getHostAddress();
logger.config("InetAddress master Host = " + master);
logger.config("InetAddress master address = " + masterAddress);
} catch (Exception e) {
logger.severe("CmdService <init> failed " + e);
logger.log(Level.FINE, "Exception", e);
}
cmds = this;
}
/**
* This method is the only way that an external object
* can get a reference to the singleton CmdService.
* This method should not be used outside engine.
* @return reference to the single CmdService
*/
public static CmdService getHandle() {
return cmds;
}
/**
* Obtains the name of the master machine.
* @return The master machine name
*/
public String getMaster() {
return master;
}
/**
* Returns the ip address of the master.
* @return The ip address of the master
*/
public String getMasterIP() {
return masterAddress;
}
/**
* Returns the ip address of the master's interface best used for
* communicating with the target host.
* @param agentHost The target host
* @return The ip address of the master
*/
public String getMasterIP(String agentHost) {
return ifMap.get(agentHost);
}
/**
*
* This method is called after every run to re-initialize the data
* structures that need to change from one run to another.
*
*/
public void init() {
machinesList.clear();
cmdp.clear();
filep.clear();
hostInterfaces.clear();
}
/**
* This method initializes the CmdAgent RMI server processes
* on the specified set of machines.
* This method can be called multiple times to initialize multiple
* classes of machines.
* @param benchName The name of the benchmark
* @param par The parameter repository
* @return true if successful, false if setup failed
*/
public boolean setup(String benchName, ParamRepository par) {
String home = par.getParameter("fh:jvmConfig/fh:home");
if (home != null)
home = home.trim();
if (home == null || home.length() == 0) {
home = Utilities.getJavaHome();
logger.config("JAVA_HOME set to " + home);
}
if(!(new File(home)).isDirectory()) {
logger.severe("Cannot set JAVA_HOME. " + home +
" is not a valid JAVA_HOME. Exiting");
return false;
}
javaHome = home;
// Check whether the target JVM supports -XX:+DisableExplicitGC or not.
String egc = "-XX:+DisableExplicitGC";
Command probeCmd = new Command(javaHome + File.separator + "bin" +
File.separator + "java", egc, "-version");
probeCmd.setLogLevel(Command.STDOUT, Level.FINER);
probeCmd.setLogLevel(Command.STDERR, Level.FINER);
try {
CommandHandle handle = probeCmd.execute();
if (handle.exitValue() != 0)
egc = null;
} catch (IOException e) {
egc = null;
} catch (InterruptedException e) {
}
final String disableEGC = egc;
// We need to be careful to escape properties having '\\' on win32
String escapedHome = Config.FABAN_HOME.replace("\\", "\\\\");
String fs = File.separatorChar == '\\' ? "\\\\" : File.separator;
jvmOptions = new ArrayList<String>();
jvmOptions.add("-Dfaban.home=" + escapedHome);
jvmOptions.add("-Djava.security.policy=" + escapedHome + "config" +
fs + "faban.policy");
jvmOptions.add("-Djava.util.logging.config.file=" + escapedHome +
"config" + fs + "logging.properties");
jvmOptions.add("-Dfaban.registry.port=" + Config.RMI_PORT);
jvmOptions.add("-Dfaban.logging.port=" + Config.LOGGING_PORT);
try {
// Update the logging.properties file in config dir
Properties log = new Properties();
FileInputStream in = new FileInputStream(Config.CONFIG_DIR +
"logging.properties");
log.load(in);
in.close();
// Update if it has changed.
if (!(log.getProperty("java.util.logging.SocketHandler.host").
equals(master) &&
log.getProperty("java.util.logging.SocketHandler.port").
equals(String.valueOf(Config.LOGGING_PORT)))) {
log.setProperty("java.util.logging.SocketHandler.host", master);
log.setProperty("java.util.logging.SocketHandler.port",
String.valueOf(Config.LOGGING_PORT));
FileOutputStream out = new FileOutputStream(
new File(Config.CONFIG_DIR + "logging.properties"));
log.store(out, "Faban logging properties");
out.close();
}
} catch (IOException e) {
logger.log(Level.SEVERE, "Failed to initialize CmdAgent " + e, e);
}
// Start RMI registry and Registry
try {
// Create classpath with all client jars in faban/lib dir.
// Benchmark specific stubs will be in one of the jars.
File[] libs = (new File(Config.LIB_DIR)).listFiles();
StringBuffer buf = new StringBuffer();
for (int i = 0; i < libs.length; i++) {
if (libs[i].isFile()) {
buf.append(libs[i].getAbsolutePath() + File.pathSeparator);
}
}
buf.setLength(buf.length() - 1);
if (buf.indexOf(" ") != -1) {
buf.insert(0, '"');
buf.append('"');
}
String classpath = buf.toString();
// The registry should not consume much resources. Just don't
// use the driver JVM options and set it to 32m - 1024m dynamic.
// This should not be performance sensitive at all.
List<String> cmd = new ArrayList<String>();
cmd.add(javaHome + File.separator + "bin" + File.separator +
"java");
cmd.addAll(jvmOptions);
cmd.add("-Xms32m");
cmd.add("-Xmx1024m");
cmd.add("-cp");
cmd.add(classpath);
cmd.add("com.sun.faban.common.RegistryImpl");
logger.info("Starting Registry.");
Command rmiCmd = new Command(cmd);
rmiCmd.setSynchronous(false);
rmiCmd.setLogLevel(Command.STDOUT, Level.WARNING);
registryCmd = rmiCmd.execute();
} catch (Exception e) {
logger.log(Level.SEVERE, "Couldn't start Registry. " +
"Please check if its already running", e);
return false;
}
// Now add the driver options to the JVM options. Need them after this.
String jvmOpts =
par.getParameter("fh:jvmConfig/fh:jvmOptions");
if (jvmOpts != null)
jvmOpts = jvmOpts.trim();
if((jvmOpts == null) || (jvmOpts.length() == 0))
jvmOpts = "";
List<String> usrOpts = Command.parseArgs(jvmOpts);
if (disableEGC != null && !usrOpts.contains(disableEGC))
usrOpts.add(disableEGC);
jvmOptions.addAll(usrOpts);
// RMI registry takes a bit of time to startup. So sleep for some time
try {
logger.fine("Waiting for RMI registry and Registry to startup");
Thread.sleep(10000);
} catch (InterruptedException e) {
}
try {
registry = RegistryLocator.getRegistry(Config.RMI_PORT);
} catch (Exception e) {
logger.log(Level.SEVERE, "Unable to connect to Registry.", e);
return false;
}
// an agent needs to be started on the master machine
// first since configuration of agents on other machines
// depend on a CmdAgent running on the master machine
// We need to scan the machines to ensure that they are not a different
// incarnation of the master's name. If they are, switch the master to
// use these names instead.
// Also, we use the same loop to create a non-duplicate set of remote
// machines. This is used later to find the interfaces to the remote
// machine.
InetAddress[] masterIps = null;
try {
masterIps = InetAddress.getAllByName(master);
} catch (UnknownHostException e) {
logger.log(Level.SEVERE, "Strange! Master is unknown.", e);
return false;
}
HashSet<String> remoteMachines = new HashSet<String>();
boolean isMasterSet = false;
List<ParamRepository.HostConfig> hostConfigs = null;
try {
hostConfigs = par.getHostConfigs();
} catch (ConfigurationException e) {
logger.log(Level.SEVERE, "Problem reading parameter file", e);
}
outer:
for (ParamRepository.HostConfig hostConfig : hostConfigs) {
String[] machines = hostConfig.hosts;
for (int i = 0; i < machines.length; i++) {
// Check for no localhost, we don't allow it.
if (machines[i].startsWith("localhost")) {
if (machines[i].length() == 9 || // localhost
machines[i].charAt(9) == '.') { // localhost.domain
logger.severe("Host names must not be localhost. " +
"Please use real host names or IP addresses " +
"instead. Terminating run!");
return false;
}
}
try {
InetAddress[] machineIps =
InetAddress.getAllByName(machines[i]);
if (sameHost(masterIps, machineIps)) {
if (!isMasterSet) { // Set the master to the first
// found master name in the list.
master = machines[i];
isMasterSet = true;
} else { // Set all subsequent masters to the same.
machines[i] = master;
}
} else { // All remote machines go into a set.
remoteMachines.add(machines[i]);
}
} catch (UnknownHostException e) {
logger.log(Level.WARNING, machines[i] + " is unknown.", e);
}
}
}
// Next we use the command map to get the right
// rsh command based on the undelying OS.
try {
binMap = CmdMap.getCmdMap(null);
rsh = binMap.get("rsh");
agent = binMap.get("agent");
} catch (Exception e) {
logger.log(Level.WARNING, "Failed to obtain command map.", e);
}
if (rsh == null) {
rsh = new ArrayList<String>();
rsh.add("rsh");
}
//only case in which interfaceAddress is not an address but
//the hostname of the master machine. used in CmdAgentImpl
//the cmdagent on the master machine is registered under 2
// names, Config.CMD_AGENT@master as well as just Config.CMD_AGENT
if (!machinesList.contains(master)) {
if (!startCmdAgent(benchName, master, master)) {
return false;
}
machinesList.add(master);
}
// this is necessary in case you are on a private network
// where the machine's private ip address is not the same as it's
// public ip address
// Fist check specific scripts for the arch
String scriptPath = Config.BIN_DIR + Config.ARCH_DIR + "interface";
File ifScript = new File(scriptPath.trim());
// Then check script for the OS. If it exists, use it.
// It is usually more reliable than the interface probe.
if (!ifScript.exists()) {
logger.finer("Could not find interface script at " +
ifScript.getAbsolutePath());
scriptPath = Config.BIN_DIR + Config.OS_DIR + "interface";
ifScript = new File(scriptPath.trim());
}
ifMap = new HashMap<String, String>();
boolean ifMapComplete = false;
if (ifScript.exists()) {
ifMapComplete = getIfMap(remoteMachines, ifScript, ifMap);
} else {
logger.finer("Could not find interface script at " +
ifScript.getAbsolutePath());
ifScript = null;
}
// If we have no interface script or the interface script did not
// do a complete job, we'll resort to the probe.
// Most reliable when run as root, but buggy in parallel mode.
// Also the interface probe needs JDK1.6 or later.
if (!ifMapComplete) {
if ("1.6".compareTo(System.getProperty("java.version")) > 0) {
logger.severe("Could not find a way to check the interface!");
return false;
}
InterfaceProbe iProbe = null;
try {
iProbe = new InterfaceProbe(Config.THREADPOOL);
iProbe.getIfMap(remoteMachines, ifMap);
} catch (SocketException e) {
logger.log(Level.SEVERE,
"Could not find a way to check the interface!", e);
}
}
// cycles through benchmark machines starting up agents and
// configuring them
for (ParamRepository.HostConfig hostConfig : hostConfigs) {
String[] machines = hostConfig.hosts;
for (int i = 0; i < machines.length; i++) {
// Do not start duplicate Cmd agent
if (machinesList.contains(machines[i])) {
continue;
}
String interfaceAddress = ifMap.get(machines[i]);
if (interfaceAddress == null || interfaceAddress.length() == 0) {
return false;
}
if (!startCmdAgent(benchName, machines[i], interfaceAddress)) {
return false;
}
// By adding the mach to the list we prevent multiple
// agents being started on the same server
machinesList.add(machines[i]);
}
}
try {
Thread.sleep(20000);
} catch (InterruptedException e) {
}
for (int i = 0; i < machinesList.size(); i++) {
if (!getCmdAgent((String) machinesList.get(i))) {
return false;
}
}
if (par.getBooleanValue("fa:runConfig/fh:timeSync", true)) {
setClocks();
}
return true;
}
void setHostRoles(HostRoles hr) {
hostRoles = hr;
// We need to populate the machinesList and cmdp with
// the real host names.
// First get the real names.
String[] realHosts = hostRoles.getHostsInOrder();
// For each real host name not in machinesList, we take the first alias
// and look it up in the machinesList. Fetch the command agent and add
// the real name to the machinesList -> cmdp mapping.
for (String hostName : realHosts) {
if (!machinesList.contains(hostName)) {
String[] aliases = hostRoles.getAliasesByHost(hostName);
CmdAgent a = findCmdAgent(aliases[0]); // Just one is enough
machinesList.add(hostName);
cmdp.add(a);
}
}
}
private boolean getIfMap(Collection<String> hosts, File ifScript,
Map<String, String> ifMap) {
boolean complete = true;
for (String host : hosts) {
String interfaceAddress = null;
String ifCommand = ifScript.getAbsolutePath() + ' ' + host;
logger.fine("Detecting interface: " + ifCommand);
try {
Process p = Runtime.getRuntime().exec(ifCommand);
BufferedReader bufR = new BufferedReader(
new InputStreamReader(p.getInputStream()));
interfaceAddress = bufR.readLine();
if (interfaceAddress != null) {
interfaceAddress = interfaceAddress.trim();
if ("127.0.0.1".equals(interfaceAddress)) {
complete = false;
ifMap.put(host, "");
} else {
ifMap.put(host, interfaceAddress);
}
}
int exitValue = -1;
if (interfaceAddress != null &&
interfaceAddress.length() > 0) { //Read something...
exitValue = p.waitFor();
if (exitValue != 0) {
logger.warning("interface: Cannot reach system " +
host);
complete = false;
ifMap.put(host, "");
continue;
}
} else { // Nothing read, check stderr
bufR = new BufferedReader(
new InputStreamReader(p.getErrorStream()));
logger.severe(host + ": " + bufR.readLine());
ifMap.put(host, "");
continue;
}
} catch (Exception e) {
logger.log(Level.SEVERE,
"Error in executing the interface program: " +
ifCommand, e);
break;
}
logger.config("Interface Address = " + interfaceAddress);
}
return complete;
}
private boolean getCmdAgent(String mach) {
try {
String s = Config.CMD_AGENT + "@" + mach;
logger.fine("CmdService: Connecting to " + s);
int retry = 1;
CmdAgent c = (CmdAgent) registry.getService(s);
for (; c == null && retry <= 10; retry++) {
Thread.sleep(10000);
logger.warning("Retry connecting to " + s + ", count " +
retry + '.');
c = (CmdAgent) registry.getService(s);
}
if (c == null) {
logger.severe("Could not connect to " + s);
return (false);
}
cmdp.add(c);
/* Note the agent registration process:
* 1. Create and register the command agent.
* 2. Download benchmark code
* 3. Create the lib classpath
* 4. Create and register file agent
* So it may take quite some time between the registration of
* the command agent and the file agent. But we can be pretty
* sure it'll happen. So just wait. Timeout after 100 retries.
*/
s = Config.FILE_AGENT + "@" + mach;
logger.fine("FileService: Connecting to " + s);
retry = 1;
FileAgent f = (FileAgent) registry.getService(s);
for (; f == null && retry <= 100; retry++) {
Thread.sleep(1000);
logger.fine("Retry obtaining file service from " + s +
", count " + retry + '.');
f = (FileAgent) registry.getService(s);
}
if (f == null) {
logger.severe("Timed out obtaining file service from " + s);
return (false);
}
filep.add(f);
// Added by Ramesh to get the real hostnames of the servers
logger.info("CmdService: Configured " + s + " on server " +
c.getHostName());
return true;
} catch (Exception e) {
logger.log(Level.SEVERE, "Error accessing command agent on system " + mach, e);
return (false);
}
}
/* start up the CmdAgent applications
* We use a script 'cmd' which will setup the CLASSPATH before
* invoking CmdAgent
*/
private boolean startCmdAgent(String benchName, String mach,
String interfaceAddress) {
hostInterfaces.setProperty(mach, interfaceAddress);
List<String> cmd = new ArrayList<String>();
List<String> agentParams = new ArrayList<String>();
agentParams.add(mach);
agentParams.add(interfaceAddress);
agentParams.add(masterAddress);
agentParams.add(javaHome);
agentParams.addAll(jvmOptions);
agentParams.add("faban.benchmarkName=" + benchName);
try {
if (mach.equals(master)) {
cmd.addAll(agent);
cmd.addAll(agentParams);
logger.fine("Executing " + cmd);
Command cmdAgent = new Command(cmd);
cmdAgent.setSynchronous(false);
cmdAgent.setLogLevel(Command.STDOUT, Level.WARNING);
cmdAgent.execute();
} else { // if the machine is not the master machine, we need to
// do an rsh or talk to the agent daemon and pass download
// instructions.
// Many times, the FABAN_URL cannot be reached by the benchmark
// downloader. So it is better to change the URL to access
// the master via the best interface, by ip address instead of
// host name.
URL fabanURL = new URL(Config.FABAN_URL);
URL downloadURL = new URL(fabanURL.getProtocol(),
interfaceAddress, fabanURL.getPort(),
fabanURL.getFile());
agentParams.add("faban.download=" + downloadURL.toString());
boolean agentStarted = false;
try { // See first whether we have an agent daemon.
Socket socket = new Socket(mach, Config.AGENT_PORT);
ObjectOutputStream socketOut =
new ObjectOutputStream(socket.getOutputStream());
InputStream socketIn = socket.getInputStream();
byte[] buffer = new byte[1024];
ArrayList<String> agentExtParams =
new ArrayList<String>(agentParams);
agentExtParams.add(File.separator);
agentExtParams.add(File.pathSeparator);
socketOut.writeObject(agentExtParams);
int length = socketIn.read(buffer);
socketIn.close();
socketOut.close();
socket.close();
String response = new String(buffer, 0, length);
int rcode = Integer.parseInt(response.substring(0, 3));
switch (rcode) {
case 200:
agentStarted = true;
logger.fine("Found Agent(daemon)@" + mach +
". Registering agent.");
break;
case 500:
logger.warning("Agent(daemon)@" + mach +
": " + response +
" Please report the issue " +
"and provide logs from " + mach +
":FABAN_HOME/logs/agent.log");
break;
case 409:
logger.severe("Agent(daemon)@" + mach +
": " + response);
// We do not fall back in the conflict case.
return false;
default:
logger.warning("Agent(daemon)@" + mach +
": " + response);
}
} catch (ConnectException e) {
// We should get a ConnectException if the agent was not
// started in daemon mode. This should take no time.
logger.log(Level.FINER, "Agent(daemon)@" + mach + ": " +
e.getMessage() + ". Will try remote shell instead.", e);
} catch (IOException e) {
logger.log(Level.WARNING, "Agent(daemon)@" + mach + ": " +
e.getMessage() + ". Will try remote shell instead.", e);
}
if (!agentStarted) {
cmd.clear();
cmd.addAll(rsh);
cmd.add(mach);
cmd.addAll(agent);
cmd.addAll(agentParams);
Command cmdAgent = new Command(cmd);
cmdAgent.setSynchronous(false);
cmdAgent.setLogLevel(Command.STDOUT, Level.WARNING);
cmdAgent.execute();
}
}
return true;
} catch (Exception e) {
logger.log(Level.SEVERE, "Could not execute " + agent +
"on machine " + mach, e);
return false;
}
}
private boolean sameHost(InetAddress[] host1, InetAddress[] host2) {
for (int i = 0; i < host1.length; i++) {
for (int j = 0; j < host2.length; j++) {
if (host1[i].equals(host2[j])) {
return true;
}
}
}
return false;
}
private void setClocks() {
SimpleDateFormat dateFormat = new SimpleDateFormat("MMddHHmmyyyy.ss");
dateFormat.setTimeZone(new SimpleTimeZone(0, "GMT")); // Use GMT.
HashSet<String> hostSet = new HashSet<String>();
ArrayList<NameValuePair<Future<Boolean>>> tasks =
new ArrayList<NameValuePair<Future<Boolean>>>();
hostSet.add(master); // Don't try to set clock for master.
for (Object o : cmdp) {
CmdAgent agent = (CmdAgent) o;
String hostName = null;
try {
hostName = agent.getHostName();
if (hostSet.add(hostName)) {
NameValuePair<Future<Boolean>> future =
new NameValuePair<Future<Boolean>>();
future.name = hostName;
future.value = Config.THREADPOOL.submit(
new setClockTask(agent, hostName, dateFormat));
tasks.add(future);
}
} catch (RemoteException e) {
logger.log(Level.WARNING,
"Cannot communicate to agent to set time.", e);
}
}
for (NameValuePair<Future<Boolean>> future : tasks) {
try {
future.value.get(300, TimeUnit.SECONDS);
} catch (TimeoutException e) {
logger.log(Level.WARNING, "Timed out setting clock for " +
future.name);
} catch (Throwable t) {
Throwable cause = t.getCause();
while (cause != null) {
t = cause;
cause = t.getCause();
}
logger.log(Level.WARNING, t.getMessage(), t);
}
}
}
static class setClockTask implements Callable<Boolean> {
public static final long ACCURACY = 10l; // plus-minus 10ms.
CmdAgent agent;
String hostName;
SimpleDateFormat dateFormat;
setClockTask(CmdAgent agent, String hostName,
SimpleDateFormat dateFormat) {
this.agent = agent;
this.hostName = hostName;
this.dateFormat = dateFormat;
}
public Boolean call() throws IOException, InterruptedException {
// 1. If we're within accuracy, don't set the clock
long ms = System.currentTimeMillis();
long timeDiff = -agent.getTime() +
ms + (System.currentTimeMillis() - ms) / 2;
if (timeDiff < ACCURACY && timeDiff > -ACCURACY) {
logger.fine("Time difference of " + timeDiff +
" ms already in range. No need to set clock.");
return true;
}
logger.info("Time difference to host " + hostName + " is " +
timeDiff + " ms. Attempting to set clock.");
int lag = 100; // Start with 100ms latency.
int wakeBefore = 20;
// 2. Wait till we're latency/2 from second boundary
// Find next second boundary.
long nextSec;
String nextSecString = "";
long callTime;
for (int i = 0;; i++) {
if (i >= 20) {
logger.warning(hostName + " cannot accurately set remote " +
"time after " + i + " attempts. There is still a " +
"difference of " + timeDiff + " ms. Giving up.");
return false;
}
findBoundaryLoop:
for (int j = 0;; j++) {
if (j >= 20) {
logger.warning(hostName + " cannot scan time to set " +
"clock after " + j + " retries. Giving up " +
"setting clock. System may be overloaded or " +
"JVM doing too much garbage collections.");
return false;
}
logger.finer("Lag time: " + lag + "ms");
for (;;) {
ms = System.currentTimeMillis();
nextSec = (long) Math.ceil(ms / 1000d);
// We should be 100 ms from the boundary, at least.
if (nextSec * 1000 - ms < 100) {
++nextSec; // If not, we go to the next sec.
}
// Convert nextSec back to millis
nextSec *= 1000l;
callTime = nextSec - lag;
// DateFormat got passed to us and gets shared between
// multiple threads. So we need to sync.
synchronized (dateFormat) {
nextSecString = dateFormat.format(
new Date(nextSec));
}
// Now, sleep and wake up 20ms before the wanted second
// boundary. This is to avoid late calls as sleep may
// have up to 10ms wakeup delay.
long sleepTime = callTime - wakeBefore -
System.currentTimeMillis();
if (sleepTime > 0) {
Thread.sleep(sleepTime);
}
if (System.currentTimeMillis() >= callTime - 2) {
wakeBefore += wakeBefore;
if (wakeBefore > 700) {
logger.warning(hostName + " wakeup-before " +
"time reached 700ms limit. System is " +
"too busy. Giving up.");
return false;
}
continue;
}
break;
}
// Now within 20ms from the call, wait in a tight loop.
for (;;) {
long currentTime = System.currentTimeMillis();
if (currentTime == callTime) {
break findBoundaryLoop;
} else if (currentTime > callTime) {
logger.finer(hostName + "missed preset callTime " +
"of " + callTime + ". Current time is " +
currentTime + ".");
continue findBoundaryLoop; // Missed second boundary
}
}
}
// 3. Call agent to set time
ms = System.currentTimeMillis();
agent.setTime(nextSecString);
logger.finer("Actual setTime took " +
(System.currentTimeMillis() - ms) + " ms.");
// 4. Verify that time has been set properly.
ms = System.currentTimeMillis();
timeDiff = -agent.getTime() +
ms + (System.currentTimeMillis() - ms) / 2;
if (timeDiff < ACCURACY && timeDiff > -ACCURACY) {
logger.info("Setting time succeeded for " + hostName +
" after " + i + " retries. Time difference is " +
timeDiff + " ms.");
break;
} else {
logger.finer("Too large time difference of " + timeDiff +
" ms to " + hostName + ". Only " + ACCURACY +
" ms are allowed.");
lag += timeDiff;
}
}
return true;
}
}
/**
* Obtains the cached HostType object. Note that this is not a public API.
* @return The cached HostType object;
*/
public HostRoles getHostRoles() {
return hostRoles;
}
/**
* Returns the hostname of this machine as known to the machine
* itself. This method is included in order to solve a Naming problem
* related to the names of the tpcw result files to be transferred to the
* the master machine.
* @param machineName The target machine to check the host name
* @return The host name of the remote machine
*/
public String getHostName(String machineName) {
int index = machinesList.indexOf(machineName);
if (index < 0) {
return machineName; // Cannot resolve
}
String retVal = null;
try {
retVal = cmdp.get(index).getHostName();
} catch (RemoteException re) {
logger.severe("RemoteException " +
re.getCause());
logger.log(Level.FINE, "Exception", re);
}
if (retVal == null) {
return machineName;
}
return retVal;
}
/**
* Updates the paths in the local command agent.
* @param pathList The list of paths to download
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public void updatePaths(List<String> pathList)
throws RemoteException {
findCmdAgent(master).updatePaths(pathList);
}
/**
* Downloads files used by deploy images, especially services and tools.
* The pathList contains a list of resources in the form type/resource.
* @param machine The host name to initiate the download
* @param pathList The list of paths to download
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public void downloadServices(String machine, List<String> pathList)
throws RemoteException {
findCmdAgent(machine).downloadServices(pathList);
}
/**
* Returns the location of this command on the master system.
* Similar to the which shell command, 'which' returns the actual path
* to the given command. If it maps to a series of commands, they will
* be returned as a single string separated by spaces. Note that 'which'
* does not actually try to check the underlying system for commands
* in the search path. It only checks the Faban infrastructure for
* existence of such a command.
* @param cmd The command to search for
* @param svcPath The service path, if any
* @return The actual command to execute, or null if not found.
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public String which(String cmd, String svcPath) throws RemoteException {
return which(master, cmd, svcPath);
}
/**
* Returns the location of this command on the target system.
* Similar to the which shell command, 'which' returns the actual path
* to the given command. If it maps to a series of commands, they will
* be returned as a single string separated by spaces. Note that 'which'
* does not actually try to check the underlying system for commands
* in the search path. It only checks the Faban infrastructure for
* existence of such a command.
* @param machine The machine to search
* @param cmd The command to search for
* @param svcPath The service path, if any
* @return The actual command to execute, or null if not found.
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public String which(String machine, String cmd, String svcPath)
throws RemoteException {
return findCmdAgent(machine).which(cmd, svcPath);
}
/**
* Returns the location of this command on the target systems.
* Similar to the which shell command, 'which' returns the actual path
* to the given command. If it maps to a series of commands, they will
* be returned as a single string separated by spaces. Note that 'which'
* does not actually try to check the underlying system for commands
* in the search path. It only checks the Faban infrastructure for
* existence of such a command.
* @param machines The machines to search
* @param cmd The command to search for
* @param svcPath The service path, if any
* @return The actual command paths to execute, or null elements if not found.
*/
public String[] which(String[] machines, String cmd, String svcPath) {
String[] paths = new String[machines.length];
for (int i = 0; i < machines.length; i++)
try {
paths[i] = which(machines[i], cmd, svcPath);
} catch (RemoteException e) {
logger.warning("Error searching for command " + cmd + " on " +
machines[i] + '.');
}
return paths;
}
/**
* Executes a command from the master's command agent.
* @param c The command to be executed
* @param svcPath
* @return A handle to the command
* @throws IOException Error communicating with resulting process
* @throws InterruptedException Thread got interrupted waiting
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public CommandHandle execute(Command c, String svcPath)
throws IOException, InterruptedException {
return execute(master, c, svcPath);
}
/**
* Executes a command from the remote command agent.
* @param machine The target machine to execute the command
* @param c The command to be executed
* @param svcPath The location of the invoking service, if any
* @return A handle to the command
* @throws IOException Error communicating with resulting process
* @throws InterruptedException Thread got interrupted waiting
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public CommandHandle execute(String machine, Command c, String svcPath)
throws IOException, InterruptedException {
return findCmdAgent(machine).execute(c, svcPath);
}
/**
* Executes a command from the remote command agent.
* @param machines The target machines to execute the command
* @param c The command to be executed
* @param svcPath The location of the invoking service, if any
* @return Handles to the command on each of the target machines
* @throws IOException Error communicating with resulting process
* @throws InterruptedException Thread got interrupted waiting
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public CommandHandle[] execute(String[] machines, Command c, String svcPath)
throws IOException, InterruptedException {
CommandHandle[] result = new CommandHandle[machines.length];
for (int i = 0; i < machines.length; i++)
result[i] = findCmdAgent(machines[i]).execute(c, svcPath);
return result;
}
/**
* Executes a java command from the master's command agent.
* @param c The command to be executed
* @param svcPath The location of the invoking service, if any
* @return A handle to the command
* @throws IOException Error communicating with resulting process
* @throws InterruptedException Thread got interrupted waiting
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public CommandHandle java(Command c, String svcPath)
throws IOException, InterruptedException {
return java(master, c, svcPath);
}
/**
* Executes a java command from the remote command agent.
* @param machine The target machine to execute the command
* @param c The command to be executed
* @param svcPath The location of the invoking service, if any
* @return A handle to the command
* @throws IOException Error communicating with resulting process
* @throws InterruptedException Thread got interrupted waiting
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public CommandHandle java(String machine, Command c, String svcPath)
throws IOException, InterruptedException {
return findCmdAgent(machine).java(c, svcPath);
}
/**
* Executes a java command from the remote command agent.
* @param machines The target machines to execute the command
* @param c The command to be executed
* @param svcPath The location of the invoking service, if any
* @return Handles to the command on each of the target machines
* @throws IOException Error communicating with resulting process
* @throws InterruptedException Thread got interrupted waiting
* @throws RemoteException If there is a communication error to the
* remote agent
*/
public CommandHandle[] java(String[] machines, Command c, String svcPath)
throws IOException, InterruptedException {
CommandHandle[] result = new CommandHandle[machines.length];
for (int i = 0; i < machines.length; i++)
result[i] = findCmdAgent(machines[i]).java(c, svcPath);
return result;
}
/**
* Executes a job in a remote command agent.
* @param machine The host to execute the command
* @param callable The job
* @param svcPath The location of the invoking service, if any
* @return The return value of the job
* @throws Exception An error occured executing the remote job
*/
public <V extends Serializable> V execute(String machine,
RemoteCallable<V> callable,
String svcPath)
throws Exception {
return findCmdAgent(machine).exec(callable, svcPath);
}
/**
* Executes a job on remote command agents on a list of systems.
* @param machines The host names to execute the job
* @param callable The job
* @param svcPath The location of the invoking service, if any
* @return The return values of the job, in sequence
* @throws Exception An error occurred executing the job
*/
public <V extends Serializable> List<V> execute(String[] machines,
RemoteCallable<V> callable,
String svcPath)
throws Exception {
ArrayList<V> rl = new ArrayList<V>();
for (int i = 0; i < machines.length; i++)
rl.add(findCmdAgent(machines[i]).exec(callable, svcPath));
return rl;
}
/**
* Start the agent on a single machine.
* @param machine on which command should be started
* @param agentClass Impl Class of the agent to be started
* @param identifier to identify this agent later
* @return true if the command completed successfully, else false
* @throws Exception An error occurred starting the command
*/
public boolean startAgent(String machine, Class agentClass,
String identifier) throws Exception {
String m[] = new String[1];
m[0] = machine;
return (startAgent(m, agentClass, identifier));
}
/**
* Start Agent in the specified machines.
*
* @param machines on which command should be started
* @param agentClass Impl Class of the agent to be started
* @param identifier to identify this agent later
* @return true if all commands completed successfully, else false
* @throws Exception An error occurred starting the commands
*/
public boolean startAgent(String machines[], Class agentClass,
String identifier) throws Exception {
boolean result = true;
for (int i = 0; i < machines.length; i++) {
if ((machines[i] == null) || (machines[i].equals(""))) {
continue;
}
//Change the identifier to agent@host
result = result && findCmdAgent(machines[i]).
startAgent(agentClass, identifier + "@" + machines[i]);
}
return result;
}
/**
* Gets a property from a given file.
* @param machine The machine name
* @param propFile The property file name
* @param propName The property key name
* @return The property value
* @throws java.io.IOException If there is an error accessing the config file
*/
public String getProperty(String machine, String propFile, String propName)
throws IOException {
return findFileAgent(machine).getProperty(propFile, propName);
}
/**
* Kill all commands currently running and cleanup.
* This method is called when a run must be aborted
* or at the end of a benchmark run.
*/
public void kill() {
int i = 0;
try {
for (i = 0; i < cmdp.size(); i++) {
logger.info("killing CmdAgent@" + machinesList.get(i));
cmdp.get(i).kill();
}
cmdp.clear();
filep.clear();
} catch (Exception e) {
logger.log(Level.SEVERE, "Kill Failed for CmdAgent@" +
machinesList.get(i), e);
} finally {
//Exiting Registry
if (registryCmd != null) {
int retry = 0;
for (; retry < 20; retry++) {
try {
registryCmd.destroy();
Thread.sleep(1000);
int exitValue = registryCmd.exitValue();
logger.finer("Registry exited with exit value " +
exitValue + '.');
break;
} catch (InterruptedException e) {
logger.log(Level.WARNING, "Interrupted waiting for " +
"registry to terminate. " +
"Cannot verify termination status.", e);
} catch (RemoteException e) {
logger.log(Level.SEVERE, "Caught RemoteException on " +
"local CommandHandle destroy for Registry. " +
"Please report bug.", e);
} catch (IllegalThreadStateException e) {
logger.log(Level.FINER,
"Registry did not terminate! ", e);
}
}
if (retry == 20) {
logger.severe("Registry did not terminate " +
"after 20 termination attempts, giving up! " +
"Subsequent runs may have problems.");
}
}
}
}
/**
* Pushes a local file on the Faban master to the remote host.
* @param srcfile The source file name, relative to the out dir
* @param destmachine The destination machine
* @param destfile The destination file name
* @return true if successful, false otherwise
*/
public synchronized boolean push(String srcfile,
String destmachine, String destfile) {
int didx = machinesList.indexOf(destmachine);
if (didx == -1) {
throw new FabanHostUnknownException(
"Host " + destmachine + " not found!");
}
// Ensure the file is accessed from the right place.
File src = new File(srcfile);
if (!src.isAbsolute())
src = new File(Config.OUT_DIR, srcfile);
srcfile = src.getAbsolutePath();
try {
String srcHost = InetAddress.getLocalHost().getHostName();
String destHost = hostRoles.getHostByAlias(destmachine);
if (destHost.equals(srcHost)) {
if (srcfile.equals(destfile)) {
return true;
} else {
return FileHelper.copyFile(srcfile, destfile, true);
}
}
} catch (UnknownHostException e) {
logger.log(Level.SEVERE, "CmdService: Cannot determine own" +
"host name", e);
return false;
}
FileAgent destf = filep.get(didx);
try {
FileTransfer transfer = new FileTransfer(srcfile, destfile);
logger.fine("Transferring " + transfer.getSource() + "->" +
transfer.getDest() + " size " +
transfer.getSize() + " bytes.");
if (destf.push(transfer) != transfer.getSize()) {
throw new IOException("Invalid transfer size");
}
} catch (RemoteException e) {
Throwable t = e;
Throwable cause = t.getCause();
while (cause != null) {
t = cause;
cause = t.getCause();
}
logger.log(Level.SEVERE, "CmdService: Pushing - " +
"exception writing file " + destfile, t);
return false;
} catch (IOException e) {
logger.log(Level.SEVERE, "CmdService: Pushing - " +
"exception reading file " + srcfile, e);
return false;
}
return true;
}
/**
* Gets a remote file to the Faban master.
* @param srcmachine The source machine
* @param srcfile The source file name
* @param destfile The destination file name, always full path
* @return true if successful, false otherwise
*/
public synchronized boolean get(String srcmachine, String srcfile,
String destfile) {
int sidx = machinesList.indexOf(srcmachine);
if (sidx == -1) {
throw new FabanHostUnknownException(
"Host " + srcmachine + " not found!");
}
try {
String src = InetAddress.getLocalHost().getHostName();
String dest = hostRoles.getHostByAlias(srcmachine);
if (dest.equals(src)) {
if (srcfile.equals(destfile)) {
return true;
} else {
return FileHelper.copyFile(srcfile, destfile, true);
}
}
} catch (UnknownHostException e) {
logger.log(Level.SEVERE, "CmdService: Cannot determine own" +
"host name", e);
return false;
}
FileAgent srcf = filep.get(sidx);
try {
FileTransfer transfer = srcf.get(srcfile, destfile);
if (transfer.getSize() != transfer.getTransferSize()) {
throw new IOException("Received " + transfer.getSource() +
"->" + transfer.getDest() + ", " +
transfer.getTransferSize() + " out of " +
transfer.getSize() + " bytes");
}
} catch (RemoteException e) {
Throwable t = e;
Throwable cause = t.getCause();
while (cause != null) {
t = cause;
cause = t.getCause();
}
logger.log(Level.SEVERE, "CmdService: Getting - " +
"exception reading file " + srcfile, t);
return false;
} catch (IOException e) {
logger.log(Level.SEVERE, "CmdService: Getting - " +
"exception reading " + srcfile, e);
return false;
}
return true;
}
/**
* Copy a file from one remote machine to another
* This method essentially does the work of 'rcp'
* using the FileAgents on the machines.
* @param srcmachine - Name of source machine
* @param destmachine - Name of destination machine
* @param srcfile - Name of source file
* @param destfile - Name of destination file
* @param append to dest file or overwrite it
* @return true/false if copy was successful/failed
* @deprecated
*/
@Deprecated public synchronized boolean copy(String srcmachine, String destmachine,
String srcfile, String destfile,
boolean append) {
FileAgent srcf, destf = null;
FileService srcfilep = null, destfilep = null;
int sidx = machinesList.indexOf(srcmachine);
int didx = machinesList.indexOf(destmachine);
byte[] buf;
if (sidx == didx && srcfile.equals(destfile)) {
return (true);
}
if (srcfile.equals(destfile)) {
try {
String dest = cmdp.get(didx).getHostName();
String src = cmdp.get(sidx).getHostName();
if (dest == src) {
return true;
}
} catch (Exception e) {
logger.severe("CmdService: Copying - CmdAgent getHostName exception");
logger.log(Level.FINE, "Exception", e);
}
}
logger.fine("CmdService: Copying " + srcfile + " from " + srcmachine + " to " + destfile + " in " + destmachine);
srcf = filep.get(sidx);
destf = filep.get(didx);
try {
srcfilep = srcf.open(srcfile, FileAgent.READ);
if (append) {
destfilep = destf.open(destfile, FileAgent.APPEND);
} else {
destfilep = destf.open(destfile, FileAgent.WRITE);
}
// Read from src and write to dest.
buf = srcfilep.read();
destfilep.write(buf);
srcfilep.close();
destfilep.close();
} catch (Exception ie) {
logger.log(Level.WARNING, "CmdService: Could not copy " +
srcmachine + ":" + srcfile + " to " + destmachine + ":" +
destfile, ie);
return (false);
}
return true;
}
/**
* Obtains the temporary dircteroy for the given machine.
*
* @param machine The machine name
* @return The temporary directory to use on the machine
*/
public String getTmpDir(String machine) {
try {
return findCmdAgent(machine).getTmpDir();
} catch (RemoteException e) {
throw new RuntimeException(e);
}
}
private CmdAgent findCmdAgent(String machine) {
if (machine == null || machine.length() == 0) {
throw new IllegalArgumentException(
"Machine cannot be null or zero length");
}
int index = machinesList.indexOf(machine);
if (index == -1) {
throw new FabanHostUnknownException(
"Host " + machine + " not found!");
}
return cmdp.get(index);
}
private FileAgent findFileAgent(String machine) {
if (machine == null || machine.length() == 0) {
throw new IllegalArgumentException(
"Machine cannot be null or zero length");
}
int index = machinesList.indexOf(machine);
if (index == -1) {
throw new FabanHostUnknownException(
"Host " + machine + " not found!");
}
return filep.get(index);
}
/**
* Deletes the file from the machine.
*
* @param srcmachine The machine name
* @param srcfile The file name
* @return true if the file has been deleted, false otherwise
*/
public synchronized boolean delete(String srcmachine, String srcfile) {
try {
return findFileAgent(srcmachine).removeFile(srcfile);
} catch (Exception ie) {
logger.severe("CmdService: Could not delete " + srcmachine +
":" + srcfile);
logger.log(Level.FINE, "Exception", ie);
return false;
}
}
/**
* Deletes the file from the machine based on the filter provided.
*
* @param srcmachine The machine name
* @param dir The directory name
* @param filter The file filter to use
* @return true if akk files selected by the filter has been removed
*/
public synchronized boolean delete(String srcmachine, String dir,
com.sun.faban.harness.FileFilter filter) {
try {
return findFileAgent(srcmachine).removeFiles(dir, filter);
} catch (Exception ie) {
logger.severe("CmdService: Could not delete files on " +
srcmachine + ":" + dir);
logger.log(Level.FINE, "Exception", ie);
return false;
}
}
/**
* Truncates the file from the machine.
*
* @param srcmachine The machine name
* @param srcfile The file name
* @return true if the file has been deleted, false otherwise
*/
public synchronized boolean truncate(String srcmachine, String srcfile) {
try {
return findFileAgent(srcmachine).truncateFile(srcfile);
} catch (Exception ie) {
logger.severe("CmdService: Could not truncate " + srcmachine +
":" + srcfile);
logger.log(Level.FINE, "Exception", ie);
return false;
}
}
/**
* Copy a file from one remote machine to a stream on the master.
* This method essentially does the work of 'rcp'
* using the FileAgents on the machines.
* @param srcmachine - Name of source machine
* @param srcfile - Name of source file
* @param stream The stream to copy the content to
* @return true/false if copy was successful/failed
*/
public synchronized boolean copyToStream(String srcmachine, String srcfile,
OutputStream stream) {
FileService srcfilep = null;
byte[] buf = null;
FileAgent srcf = findFileAgent(srcmachine);
try {
srcfilep = srcf.open(srcfile, FileAgent.READ);
// Now loop, reading from src and writing to dest
while (true) {
buf = srcfilep.readBytes(1000000);
// logger.info(" Read " + buf);
// logger.info(buf);
// logger.info(buf.length);
stream.write(buf);
if (buf.length < 1000000) {
break;
}
}
srcfilep.close();
} catch (Exception ie) {
logger.log(Level.WARNING, "CmdService: Could not copy " +
srcmachine + ":" + srcfile, ie);
return (false);
}
return (true);
}
/**
* Set the Log level for Agents.
* @param name Logger name
* @param level Log level
*/
public void setLogLevel(String name, Level level) {
int i = 0;
try {
for (i = 0; i < cmdp.size(); i++) {
cmdp.get(i).setLogLevel(name, level);
}
} catch (Exception e) {
logger.severe(" setLogLevel Failed for CmdAgent@" + machinesList.get(i));
logger.log(Level.FINE, "Exception", e);
}
}
/**
* Obtains the registry.
* @return The registry
*/
public Registry getRegistry() {
return registry;
}
/**
* Checks whether the given remote file exists.
* @param hostName The host name to check.
* @param fileName The file name to test.
* @return true if exists, false otherwise.
*/
public boolean doesFileExist(String hostName, String fileName) {
try {
return findFileAgent(hostName).doesFileExist(fileName);
} catch (Exception ie) {
logger.log(Level.SEVERE, "CmdService: Could not check " +
hostName + ":" + fileName, ie);
return false;
}
}
/**
* Checks whether the given remote file exists and is a normal file.
* @param hostName The host name to check.
* @param fileName The file name to test.
* @return true if file is a normal file, false otherwise.
*/
public boolean isFile(String hostName, String fileName) {
try {
return findFileAgent(hostName).isFile(fileName);
} catch (Exception ie) {
logger.log(Level.SEVERE, "CmdService: Could not check " +
hostName + ":" + fileName, ie);
return false;
}
}
/**
* Checks whether the given remote file exists and is a directory.
* @param hostName The host name to check.
* @param fileName The file name to test.
* @return true if file is a directory, false otherwise.
*/
public boolean isDirectory(String hostName, String fileName) {
try {
return findFileAgent(hostName).isDirectory(fileName);
} catch (Exception ie) {
logger.log(Level.SEVERE, "CmdService: Could not check " +
hostName + ":" + fileName, ie);
return false;
}
}
}