/*
* Copyright 1999-2008 University of Chicago
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy
* of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.globus.workspace.scheduler.defaults.pilot;
import commonj.timers.TimerManager;
import edu.emory.mathcs.backport.java.util.concurrent.ExecutorService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.globus.workspace.groupauthz.GroupAuthz;
import org.globus.workspace.scheduler.NodeExistsException;
import org.globus.workspace.scheduler.NodeInUseException;
import org.globus.workspace.scheduler.NodeManagement;
import org.globus.workspace.scheduler.NodeManagementDisabled;
import org.globus.workspace.scheduler.NodeNotFoundException;
import org.globus.workspace.scheduler.defaults.ResourcepoolEntry;
import org.globus.workspace.service.binding.authorization.CreationAuthorizationCallout;
import org.nimbus.authz.AuthzDBAdapter;
import org.nimbus.authz.UserAlias;
import org.nimbustools.api.services.rm.DoesNotExistException;
import org.nimbustools.api.services.rm.ResourceRequestDeniedException;
import org.nimbustools.api.services.rm.ManageException;
import org.globus.workspace.Lager;
import org.globus.workspace.ReturnException;
import org.globus.workspace.WorkspaceConstants;
import org.globus.workspace.WorkspaceUtil;
import org.globus.workspace.WorkspaceException;
import org.globus.workspace.cmdutils.TorqueUtil;
import org.globus.workspace.persistence.WorkspaceDatabaseException;
import org.globus.workspace.scheduler.Reservation;
import org.globus.workspace.scheduler.Scheduler;
import org.globus.workspace.scheduler.defaults.NodeRequest;
import org.globus.workspace.scheduler.defaults.SlotManagement;
import org.globus.workspace.service.WorkspaceHome;
import org.globus.workspace.service.InstanceResource;
import org.globus.workspace.service.impls.site.HTTPListener;
import org.globus.workspace.service.impls.site.PilotPoll;
import org.globus.workspace.service.impls.site.SlotPollCallback;
import org.globus.workspace.xen.XenUtil;
import org.safehaus.uuid.UUIDGenerator;
import org.springframework.core.io.Resource;
import javax.sql.DataSource;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Iterator;
import java.util.List;
public class PilotSlotManagement implements SlotManagement,
SlotPollCallback,
NodeManagement {
// -------------------------------------------------------------------------
// STATIC VARIABLES
// -------------------------------------------------------------------------
private static final Log logger =
LogFactory.getLog(PilotSlotManagement.class.getName());
private static final String SEVERE_PILOT_ISSUE = "There was a severe " +
"issue with the workspace site scheduler interaction (worksapce " +
"pilot). Please contact your administrator with the time of " +
"this problem and any relevant information";
private static final String REMOTE_NODE_MGR_DISABLED = "Pilot mode: node management " +
"is not possible, use the LRM for node management";
private static final Exception SEVERE_PILOT_FAULT =
new Exception(SEVERE_PILOT_ISSUE);
private static final UUIDGenerator uuidGen = UUIDGenerator.getInstance();
private static final DateFormat localFormat = DateFormat.getDateTimeInstance();
// -------------------------------------------------------------------------
// INSTANCE VARIABLES
// -------------------------------------------------------------------------
private final Lager lager;
private final TimerManager timerManager;
private final DataSource dataSource;
private final ExecutorService sharedExecutor;
private final Object groupLock = new Object();
private WorkspaceHome instHome;
private Scheduler schedulerAdapter;
private PilotSlotManagementDB db;
private CursorPersistence cp = null;
private String sshNotifyString;
// keep reference for clean shutdown
private HTTPListener httpListener = null;
private String httpNotifyString;
private PilotPoll watcher = null;
private Pilot pilot = null;
private String logdirPath = null;
private TorqueUtil torque;
private AuthzDBAdapter authzDBAdapter;
private CreationAuthorizationCallout authzCallout;
// set from config
private String contactPort;
private String accountsPath;
private String sshNotificationInfo;
private String pilotPath;
private String pollScript;
private int grace = -1; // seconds
private int padding = -1; // seconds
private long watcherDelay = 200; // ms
private String LRM;
private String submitPath;
private String deletePath;
private String pilotVersion;
private int maxMB; // assumes homogenous nodes for now
private int ppn = -1; // assumes homogenous nodes for now
private String destination = null; // only one for now
private String extraProperties = null;
private String multiJobPrefix = null;
private String accounting;
// -------------------------------------------------------------------------
// CONSTRUCTOR
// -------------------------------------------------------------------------
public PilotSlotManagement(WorkspaceHome home,
Lager lager,
DataSource dataSource,
TimerManager timerManager,
AuthzDBAdapter authz,
CreationAuthorizationCallout authzCall) {
if (home == null) {
throw new IllegalArgumentException("home may not be null");
}
this.instHome = home;
this.sharedExecutor = home.getSharedExecutor();
if (dataSource == null) {
throw new IllegalArgumentException("dataSource may not be null");
}
this.dataSource = dataSource;
if (timerManager == null) {
throw new IllegalArgumentException("timerManager may not be null");
}
this.timerManager = timerManager;
if (lager == null) {
throw new IllegalArgumentException("lager may not be null");
}
this.lager = lager;
this.authzDBAdapter = authz;
this.authzCallout = authzCall;
}
// -------------------------------------------------------------------------
// MODULE SET (avoids circular dependency problem)
// -------------------------------------------------------------------------
public void setInstHome(WorkspaceHome homeImpl) {
if (homeImpl == null) {
throw new IllegalArgumentException("homeImpl may not be null");
}
this.instHome = homeImpl;
}
// -------------------------------------------------------------------------
// SET FROM CONFIG
// -------------------------------------------------------------------------
public void setContactPort(String contactPort) {
this.contactPort = contactPort;
}
public void setAccountsResource(Resource accountsResource) throws IOException {
this.accountsPath = accountsResource.getFile().getAbsolutePath();
}
public void setSshNotificationInfo(String info) {
if (info != null && info.trim().length() != 0) {
this.sshNotificationInfo = info;
}
}
public void setPollScriptResource(Resource pollScriptResource) throws IOException {
this.pollScript = pollScriptResource.getFile().getAbsolutePath();
}
// default is 200ms
public void setWatcherDelay(long delay) {
this.watcherDelay = delay;
}
public void setPilotPath(String pilotPath) {
this.pilotPath = pilotPath;
}
public void setGrace(int grace) {
this.grace = grace;
}
public void setPadding(int padding) {
this.padding = padding;
}
public void setLRM(String LRM) {
this.LRM = LRM;
}
public void setPpn(int ppn) {
this.ppn = ppn;
}
public void setSubmitPath(String submitPath) {
this.submitPath = submitPath;
}
public void setDeletePath(String deletePath) {
this.deletePath = deletePath;
}
public void setPilotVersion(String pilotVersion) {
this.pilotVersion = pilotVersion;
}
public void setMaxMB(int maxMB) {
this.maxMB = maxMB;
}
public void setDestination(String destination) {
if (destination != null && destination.trim().length() != 0) {
this.destination = destination;
}
}
public void setExtraProperties(String extraProperties) {
if (extraProperties != null && extraProperties.trim().length() != 0) {
this.extraProperties = extraProperties;
}
}
public void setMultiJobPrefix(String multiJobPrefix) {
if (multiJobPrefix != null && multiJobPrefix.trim().length() != 0) {
this.multiJobPrefix = multiJobPrefix;
}
}
public void setLogdirResource(Resource logdirResource) throws IOException {
this.logdirPath = logdirResource.getFile().getAbsolutePath();
}
public AuthzDBAdapter getAuthzDBAdapter() {
return authzDBAdapter;
}
public void setAuthzDBAdapter(AuthzDBAdapter authzDBAdapter) {
this.authzDBAdapter = authzDBAdapter;
}
public void setAccounting(String accounting) {
if (accounting != null && accounting.trim().length() != 0) {
this.accounting = accounting;
}
}
// -------------------------------------------------------------------------
// IoC INIT METHOD
// -------------------------------------------------------------------------
public synchronized void validate() throws Exception {
boolean httpNotificationEnabled = true;
if (this.contactPort == null) {
httpNotificationEnabled = false;
logger.info("pilot http-based notification information " +
"is not set therefore it is disabled");
}
boolean sshNotificationEnabled = true;
if (this.sshNotificationInfo == null) {
sshNotificationEnabled = false;
logger.info("pilot ssh-based (backup) notification information " +
"is not set therefore it is disabled");
}
if (sshNotificationEnabled) {
if (this.pollScript != null) {
final File pollScriptFile = new File(this.pollScript);
if (!pollScriptFile.exists() || !pollScriptFile.isFile()) {
throw new FileNotFoundException(
"Not found or not a file: '" + this.pollScript);
}
} else {
throw new Exception("pollScript setting is missing from" +
" pilot slot management configuration");
}
if (this.watcherDelay < 1) {
throw new Exception("pilot sweeper delay is less than 1, " +
"invalid");
}
if (this.watcherDelay < 50) {
logger.warn("you should probably not set sweeper delay to " +
"less than 50ms");
}
}
if (!httpNotificationEnabled && !sshNotificationEnabled) {
throw new Exception("no pilot-->container notification " +
"mechanism is set");
}
if (this.submitPath == null) {
throw new Exception("pilot LRM submit path is not set");
}
if (this.deletePath == null) {
throw new Exception("pilot LRM delete path is not set");
}
if (this.LRM == null) {
throw new Exception("pilot LRM is not set");
} else if (!this.LRM.equalsIgnoreCase("torque")) {
String x = "pilot LRM is not set to torque, the only current impl";
throw new Exception(x);
}
if (this.LRM.equalsIgnoreCase("torque")) {
this.torque = new TorqueUtil(this.submitPath,
this.deletePath);
}
if (this.maxMB <= 0) {
throw new Exception("Max guest memory is <= 0 MB. Is the " +
"configuration present (maxMB)?");
}
if (this.pilotPath == null) {
throw new Exception("path to pilot on remote nodes is not set");
} else {
if (!new File(this.pilotPath).isAbsolute()) {
throw new Exception("currently expecting path to pilot on " +
"remote nodes (pilotPath) to be an absolute path");
}
}
if (this.destination != null) {
logger.debug("Found destination: " + this.destination);
} else {
logger.debug("No destination configured.");
}
if (this.extraProperties != null) {
logger.debug("Found extra properties: " + this.extraProperties);
} else {
logger.debug("No extra properties configured.");
}
if (this.grace < 0) {
throw new Exception("grace period is less than zero, invalid. " +
"Is the configuration present?");
}
if (this.ppn < 0) {
throw new Exception("processors per node (ppn) is less than zero, " +
"invalid. Is the configuration present?");
}
if (this.padding < 0) {
throw new Exception("padding is less than zero, invalid. " +
"Is the configuration present?");
}
if (this.pilotVersion == null) {
throw new Exception("pilot version not set, there is no default");
}
// Only 0.2 is supported right now and it is never cased for
// anywhere else in this class -- will add casing in places it is
// necessary as the implementations out there diverge. At some point
// we may even remove support for old pilot versions if they cause too
// much casing to happen here or do not enable enough features such
// that too much casing or lack of important service features would
// have to happen outside this class (especially if remote client
// semantics would not be the same for all supported versions of the
// pilot).
if (this.pilotVersion.equals("0.2")) {
this.pilot = new Pilot_0_2();
} else {
throw new Exception("pilot version '" + this.pilotVersion +
"' is not supported");
}
if (sshNotificationEnabled) {
final String[] cmd = {this.pollScript};
try {
WorkspaceUtil.runCommand(cmd,
this.lager.eventLog,
this.lager.traceLog);
} catch (Exception e) {
final String err = "error testing pilot notification script: ";
// passing e to error gives very long stacktrace to user
// logger.error(err, e);
throw new Exception(err + e.getMessage());
}
this.sshNotifyString = this.sshNotificationInfo + this.pollScript;
logger.debug("tests run of pilot notification script '" +
this.pollScript + "' succeeded");
}
if (this.logdirPath != null) {
final File logdirFile = new File(this.logdirPath);
if (!logdirFile.exists()) {
throw new Exception("configured pilot log directory " +
"does not exist: " + this.logdirPath);
}
if (!logdirFile.isDirectory()) {
throw new Exception("configured pilot log directory is " +
"not a directory: " + this.logdirPath);
}
if (!logdirFile.canWrite()) {
throw new Exception("configured pilot log directory is " +
"not a directory that is writeable for this " +
"user: " + this.logdirPath);
}
} else {
throw new Exception("logdirPath setting is missing from" +
" pilot slot management configuration");
}
this.db = new PilotSlotManagementDB(this.dataSource, this.lager);
if (sshNotificationEnabled) {
this.cp = new CursorPersistence(this.db, this.timerManager, 5000);
final String eventsPath = this.pollScript + ".txt";
logger.debug("Setting events file to '" + eventsPath + "'");
this.watcher = new PilotPoll(this.timerManager,
this.lager,
this.watcherDelay,
eventsPath,
this.db.currentCursorPosition(),
this);
this.watcher.scheduleNotificationWatcher();
// this will consume pilot notifications sent during the time
// the container was down
}
if (httpNotificationEnabled) {
this.httpListener = new HTTPListener(this.contactPort,
this.accountsPath,
this,
this.sharedExecutor,
this.lager);
this.httpNotifyString = this.httpListener.getContactURL();
this.httpListener.start();
}
}
/* ************************ */
/* SlotManagement interface */
/* ************************ */
/**
* @param request a single workspace or homogenous group-workspace request
*
* @return Reservation res
* @throws ResourceRequestDeniedException exc
*/
public Reservation reserveSpace(NodeRequest request, boolean preemptable)
throws ResourceRequestDeniedException {
this.reserveSpace(request.getIds(),
request.getMemory(),
request.getCores(),
request.getDuration(),
request.getGroupid(),
request.getCreatorDN());
return new Reservation(request.getIds());
}
/**
* @param requests an array of single workspace or homogenous
* group-workspace requests
* @param coschedid coscheduling (ensemble) ID
*
* @return Reservation res
* @throws ResourceRequestDeniedException exc
*/
public Reservation reserveCoscheduledSpace(NodeRequest[] requests,
String coschedid)
throws ResourceRequestDeniedException {
if (requests == null || requests.length == 0) {
throw new IllegalArgumentException("requests null or length 0?");
}
// the LRM request will be for the highest memory and duration (the
// lesser workspaces will not get this extra memory or time -- this is
// capacity vs. mapping and we will get more sophisticated here later)
int highestMemory = 0;
int highestCores = 0;
int highestDuration = 0;
final ArrayList idInts = new ArrayList(64);
final ArrayList allDurations = new ArrayList(64);
for (int i = 0; i < requests.length; i++) {
final int thisMemory = requests[i].getMemory();
if (highestMemory < thisMemory) {
highestMemory = thisMemory;
}
final int thisCores = requests[i].getCores();
if (highestCores < thisCores) {
highestCores = thisCores;
}
final int thisDuration = requests[i].getDuration();
if (highestDuration < thisDuration) {
highestDuration = thisDuration;
}
final int[] ids = requests[i].getIds();
if (ids == null) {
throw new ResourceRequestDeniedException(
"Cannot proceed, no ids in NodeRequest parameter (?)");
}
for (int j = 0; j < ids.length; j++) {
idInts.add(new Integer(ids[j]));
allDurations.add(new Integer(thisDuration));
}
}
final int length = idInts.size();
final int[] all_ids = new int[length];
final int[] all_durations = new int[length];
for (int i = 0; i < length; i++) {
all_ids[i] = ((Number)idInts.get(i)).intValue();
all_durations[i] = ((Number)allDurations.get(i)).intValue();
}
// Assume that the creator's DN is the same for each node
final String creatorDN = requests[0].getCreatorDN();
this.reserveSpace(all_ids, highestMemory, highestCores, highestDuration, coschedid, creatorDN);
return new Reservation(all_ids, null, all_durations);
}
/**
* Only handling one slot per VM for now, will change in the future
* (multiple layers).
*
* Only handling homogenous requests for now.
*
* @param vmids array of IDs. If array length is greater than one, it is
* up to the implementation (and its configuration etc) to decide
* if each must map to its own node or not. In the case where more
* than one VM is mapped to the same node, the returned node
* assignment array will include duplicates.
* @param memory megabytes needed
* @param requestedCores needed
* @param duration seconds needed
* @param uuid group ID, can not be null if vmids is length > 1
* @param creatorDN the DN of the user who requested creation of the VM
*
* @throws ResourceRequestDeniedException can not fulfill request
*/
private void reserveSpace(final int[] vmids,
final int memory,
final int requestedCores,
final int duration,
final String uuid,
final String creatorDN)
throws ResourceRequestDeniedException {
if (vmids == null) {
throw new IllegalArgumentException("no vmids");
}
if (memory > this.maxMB) {
String msg = "Memory request (" + memory + " MB) cannot be " +
"fulfilled by any VMM node (maximum: " + this.maxMB +
" MB).";
throw new ResourceRequestDeniedException(msg);
}
// When there is no core request, the default is -1,
// we would actually like one core.
int cores;
if (requestedCores <= 0) {
cores = 1;
}
else {
cores = requestedCores;
}
if (vmids.length > 1 && uuid == null) {
logger.error("cannot make group space request without group ID");
throw new ResourceRequestDeniedException("internal " +
"pilot management error");
}
final String slotid;
if (uuid == null) {
slotid = uuidGen.generateRandomBasedUUID().toString();
} else {
slotid = uuid;
try {
for (int i = 0; i < vmids.length; i++) {
// add to our own group register, encapsulated from
// main service group/coscheduling management
this.db.newGroupMember(uuid, vmids[i]);
}
} catch (WorkspaceDatabaseException e) {
logger.error(e.getMessage(), e);
throw new ResourceRequestDeniedException("internal " +
"pilot management error");
}
}
this.reserveSpaceImpl(memory, cores, duration, slotid, vmids, creatorDN);
// pilot reports hostname when it starts running, not returning an
// exception to signal successful best effort pending slot
}
private void reserveSpaceImpl(final int memory,
final int cores,
final int duration,
final String uuid,
final int[] vmids,
final String creatorDN)
throws ResourceRequestDeniedException {
final String outputFile = this.logdirPath + File.separator + uuid;
final int dur = duration + this.padding;
final long wallTime = duration + this.padding;
// If the pbs.ppn option in pilot.conf is 0, we should send
// the number of CPU cores used by the VM as the ppn string,
// otherwise, use the defined ppn value
int ppnRequested;
if (this.ppn == 0) {
ppnRequested = cores;
}
else {
ppnRequested = this.ppn;
}
String account = getAccountString(creatorDN, this.accounting);
// we know it's torque for now, no casing
final ArrayList torquecmd;
try {
torquecmd = this.torque.constructQsub(this.destination,
memory,
vmids.length,
ppnRequested,
wallTime,
this.extraProperties,
outputFile,
false,
false,
account);
} catch (WorkspaceException e) {
final String msg = "Problem with Torque argument construction";
if (logger.isDebugEnabled()) {
logger.error(msg + ": " + e.getMessage(), e);
} else {
logger.error(msg + ": " + e.getMessage());
}
// scrubbing what client sees
throw new ResourceRequestDeniedException(msg);
}
// no casing necessary yet for pilot, only 0.2 supported now
final ArrayList pilotCommon;
try {
pilotCommon = this.pilot.constructCommon(false, false, true, null);
} catch (ManageException e) {
final String msg = "Problem with pilot argument construction";
if (logger.isDebugEnabled()) {
logger.error(msg + ": " + e.getMessage(), e);
} else {
logger.error(msg + ": " + e.getMessage());
}
// scrubbing what client sees
throw new ResourceRequestDeniedException(msg);
}
// same params sent to each pilot in a group job
final ArrayList pilotReserveSlot;
try {
final String notifyString;
if (this.httpNotifyString != null
&& this.sshNotifyString != null) {
notifyString = this.httpNotifyString + "+++" +
this.sshNotifyString;
} else if (this.httpNotifyString != null) {
notifyString = this.httpNotifyString;
} else if (this.sshNotifyString != null) {
notifyString = this.sshNotifyString;
} else {
final String msg = "No pilot-->service notification mechanism";
throw new ResourceRequestDeniedException(msg);
}
pilotReserveSlot = this.pilot.constructReserveSlot(memory,
dur,
this.grace,
uuid,
notifyString);
} catch (ManageException e) {
final String msg = "Problem with pilot argument construction";
if (logger.isDebugEnabled()) {
logger.error(msg + ": " + e.getMessage(), e);
} else {
logger.error(msg + ": " + e.getMessage());
}
// scrubbing what client sees
throw new ResourceRequestDeniedException(msg);
}
final StringBuffer pilotcmdbuf = new StringBuffer(256);
if (this.multiJobPrefix != null && vmids.length > 1) {
pilotcmdbuf.append(this.multiJobPrefix);
pilotcmdbuf.append(" ");
}
pilotcmdbuf.append(this.pilotPath);
Iterator iter = pilotCommon.iterator();
while (iter.hasNext()) {
pilotcmdbuf.append(" ");
pilotcmdbuf.append(iter.next());
}
iter = pilotReserveSlot.iterator();
while (iter.hasNext()) {
pilotcmdbuf.append(" ");
pilotcmdbuf.append(iter.next());
}
final String pilotcmd = pilotcmdbuf.toString();
logger.info("pilot command = " + pilotcmd);
final String[] cmd = (String[]) torquecmd.toArray(
new String[torquecmd.size()]);
String stdout;
try {
stdout = WorkspaceUtil.runCommand(cmd, true, pilotcmd,
this.lager.eventLog,
this.lager.traceLog);
} catch (WorkspaceException e) {
final String msg = "Problem calling Torque";
if (logger.isDebugEnabled()) {
logger.error(msg + ": " + e.getMessage(), e);
} else {
logger.error(msg + ": " + e.getMessage());
}
// scrubbing what client sees
throw new ResourceRequestDeniedException(msg);
} catch (ReturnException e) {
final String msg = "Problem calling Torque";
final StringBuffer buf = new StringBuffer(msg);
buf.append(": return code = ")
.append(e.retval);
if (e.stderr != null) {
buf.append(", stderr = '")
.append(e.stderr)
.append("'");
} else {
buf.append(", no stderr");
}
if (e.stdout != null) {
buf.append(", stdout = '")
.append(e.stdout)
.append("'");
} else {
buf.append(", no stdout");
}
logger.error(msg + ": " + buf.toString());
// scrubbing what client sees
throw new ResourceRequestDeniedException(msg);
}
if (stdout == null || stdout.length() == 0) {
final String msg = "Inexplicable problem receiving job ID from" +
" Torque (return == 0, but no stdout), aborting.";
logger.error(msg);
throw new ResourceRequestDeniedException(msg);
}
// TODO: analyze stdout here, should have no newlines or spaces
logger.debug("torque stdout = " + stdout);
stdout = stdout.trim();
if (stdout.indexOf('\n') >= 0) {
logger.warn("torque stdout has a new line");
// todo: throw exc? strip it?
}
try {
if (vmids.length == 1) {
this.db.newSlot(uuid, vmids[0], stdout, dur);
} else {
this.db.newSlotGroup(uuid, vmids, stdout, dur);
}
} catch (WorkspaceDatabaseException e) {
String msg = "Problem with service database, aborting pilot job.";
if (logger.isDebugEnabled()) {
logger.error(msg + ": " + e.getMessage(), e);
} else {
logger.error(msg + ": " + e.getMessage());
}
// we know it's torque for now, no casing
try {
WorkspaceUtil.runCommand(this.torque.constructQdel(stdout),
this.lager.eventLog,
this.lager.traceLog);
} catch (WorkspaceException e2) {
String msg2 = " (and problem with Torque qdel)";
msg += msg2;
if (logger.isDebugEnabled()) {
logger.error(msg2 + ": " + e2.getMessage(), e2);
} else {
logger.error(msg2 + ": " + e2.getMessage());
}
} catch (ReturnException e2) {
String msg2 = " (and problem calling Torque qdel)";
msg += msg2;
StringBuffer buf = new StringBuffer(msg2);
buf.append(": return code = ")
.append(e2.retval);
if (e2.stderr != null) {
buf.append(", stderr = '")
.append(e2.stderr)
.append("'");
} else {
buf.append(", no stderr");
}
if (e2.stdout != null) {
buf.append(", stdout = '")
.append(e2.stdout)
.append("'");
} else {
buf.append(", no stdout");
}
logger.error(buf.toString());
}
throw new ResourceRequestDeniedException(msg);
}
if (this.watcher != null) {
this.watcher.scheduleNotificationWatcher();
}
}
public boolean canCoSchedule() {
return true;
}
public void releaseSpace(int reservationID) throws ManageException {
if (lager.traceLog) {
logger.trace("releaseSpace(), id = " + reservationID);
}
PilotSlot slot;
try {
slot = this.db.getSlot(reservationID);
} catch (SlotNotFoundException e) {
// fine in several cases
logger.debug("slot with vmid " + reservationID + " not found");
return;
}
if (slot.partOfGroup) {
// todo: move to lock per uuid, LockManager etc.
synchronized (this.groupLock) {
// Need to retrieve again after being under lock because others
// in the group will in many cases be destroyed at once and
// there can now be situations where okToReleaseBlock can
// succeed more than once. For example this will happen in
// the situation where a pilot job is still pending with the
// LRM when a group/ensemble is destroyed.
PilotSlot aslot;
try {
aslot = this.db.getSlot(reservationID);
} catch (SlotNotFoundException e) {
return;
}
// one slot in the block has enough information to be used
// in impl of okToReleaseBlock() and releaseSpaceImpl()
if (okToReleaseBlock(aslot)) {
// force qdel for block release
this.releaseSpaceImpl(aslot, false);
} else {
logger.debug("Slot is part of block and it is not " +
"OK to release entire block yet");
}
}
} else {
this.releaseSpaceImpl(slot, slot.terminal);
}
}
public void releaseSpace(NodeRequest nodeRequest,
Reservation reservation,
boolean preemptable) throws ManageException {
if (nodeRequest == null) {
throw new IllegalArgumentException("nodeRequest may not be null");
}
if (reservation == null) {
throw new IllegalArgumentException("reservation may not be null");
}
for (int id : reservation.getIds()) {
this.releaseSpace(id);
}
}
// add release-pending and check if all other VMs in block's pilots
// have a release-pending or not
private boolean okToReleaseBlock(PilotSlot slot) throws ManageException {
try {
if (slot.nodename != null) {
this.db.setSlotPendingRemove(slot);
}
} catch (SlotNotFoundException e) {
// this should never happen because of groupLock
logger.error("inexplcable problem, block slot found but " +
"then not updateable");
}
int[] vmids = this.db.findVMsInGroup(slot.uuid);
for (int i = 0; i < vmids.length; i++) {
try {
PilotSlot aSlot = this.db.getSlot(vmids[i]);
if (aSlot.nodename == null) {
continue;
}
if (!aSlot.pendingRemove) {
return false;
}
} catch (SlotNotFoundException e) {
// again, should not happen because of groupLock
logger.error("slot for vm #" + vmids[i] + " not found?");
}
}
return true;
}
private void releaseSpaceImpl(PilotSlot slot,
boolean terminal) throws ManageException {
// If we've already received word from the pilot that it is in a
// terminal situation, no need to invoke qdel because the pilot's exit
// will end the LRM job -- unless the slot is still pending and
// therefore the pilot has not been run yet
if (!terminal || slot.pending) {
try {
WorkspaceUtil.runCommand(
this.torque.constructQdel(slot.lrmhandle),
this.lager.eventLog,
this.lager.traceLog,
slot.vmid);
} catch (WorkspaceException e) {
String msg = "Problem with Torque qdel";
if (logger.isDebugEnabled()) {
logger.error(msg + ": " + e.getMessage(), e);
} else {
logger.error(msg + ": " + e.getMessage());
}
} catch (ReturnException e) {
String msg = "Problem calling Torque qdel";
StringBuffer buf = new StringBuffer(msg);
buf.append(": return code = ")
.append(e.retval);
if (e.stderr != null) {
buf.append(", stderr = '")
.append(e.stderr)
.append("'");
} else {
buf.append(", no stderr");
}
if (e.stdout != null) {
buf.append(", stdout = '")
.append(e.stdout)
.append("'");
} else {
buf.append(", no stdout");
}
logger.error(buf.toString());
}
}
// In most situations we will hear from the pilot again about this
// slot/block (it won't be here which is expected)
this.db.removeSlot(slot.uuid);
}
public boolean isBestEffort() {
return true;
}
/**
* Strict evacuation means that the scheduler should not allow any time
* consuming action on the slot after the running duration expires (actions
* such as unpropagate).
*
* @return true if implementation requires strict evacuation
*/
public boolean isEvacuationStrict() {
return true;
}
public void setScheduler(Scheduler adapter) {
this.schedulerAdapter = adapter;
}
public boolean isNeededAssociationsSupported() {
return false;
}
/* ********************************** */
/* NotificationPollCallback interface */
/* ********************************** */
public int numPendingNotifications() throws Exception {
return this.db.numSlotsCached(false);
}
public void decreaseNumPending(int n) throws Exception {
// ignored
}
public void cursorPosition(long pos) {
if (this.cp != null) {
this.cp.cursorPosition(pos);
}
}
/* ************************** */
/* SlotPollCallback interface */
/* ************************** */
/**
* The pilot reports the slot has been successfully reserved and what host
* it's ended up running on.
*
* @param slotid uuid
* @param hostname slot node
* @param timestamp time of reservation
*/
public void reserved(String slotid, String hostname, Calendar timestamp) {
if (this.schedulerAdapter == null) {
logger.error("Severe problem, slot manager has received word " +
"that the slot '" + slotid + "' is reserved (hostname '" +
hostname + "') but the manager has " +
"not been configured with a way to inform service " +
"scheduler to proceed.");
try {
PilotSlot slot = this.getSlotAndAssignVM(slotid, hostname);
// this eventually causes this.releaseSpace() to be called
// unless there was a race
this.cancelWorkspace(slot.vmid, SEVERE_PILOT_FAULT);
} catch (SlotNotFoundException e) {
logger.error(e.getMessage());
} catch (WorkspaceDatabaseException e) {
logger.error(e.getMessage());
}
return;
}
try {
final PilotSlot slot = this.getSlotAndAssignVM(slotid, hostname);
if (hostname == null) {
logger.error("Pilot '" + slotid + "' sent reserved message " +
"without hostname (?). Cancelling vm #" + slot.vmid +
" and running trash.");
// this eventually causes this.releaseSpace() to be called
// unless there was a race
this.cancelWorkspace(slot.vmid, SEVERE_PILOT_FAULT);
return;
}
if (timestamp == null) {
logger.error("Pilot '" + slotid + "' sent reserved message " +
"without timestamp (?). Cancelling vm #" + slot.vmid +
" and running trash.");
// this eventually causes this.releaseSpace() to be called
// unless there was a race
this.cancelWorkspace(slot.vmid, SEVERE_PILOT_FAULT);
return;
}
final InstanceResource resource;
try {
resource = this.instHome.find(slot.vmid);
} catch (DoesNotExistException e) {
final String msg = "workspace #" + slot.vmid + " is unknown " +
"to the service but the pilot tracker has receieved " +
"space for it to run? pilot ID: '" + slotid + "' " +
"There is nothing we can do about this.";
logger.error(e.getMessage());
return;
}
final int runningTime =
resource.getVM().getDeployment().getMinDuration();
// double-checking assumptions
if (runningTime > slot.duration - this.padding) {
logger.error("The running time stored for workspace #" +
slot.vmid + " is greater than slot duration (?). " +
"Implementation error, backing out.");
this.cancelWorkspace(slot.vmid, SEVERE_PILOT_FAULT);
return;
}
logger.debug("reserved: running time = " + runningTime +
" (slot duration = " + slot.duration + ")");
Calendar stop = (Calendar) timestamp.clone();
stop.add(Calendar.SECOND, runningTime);
Calendar slotstop = (Calendar) timestamp.clone();
slotstop.add(Calendar.SECOND, slot.duration);
String msg = Lager.ev(slot.vmid) +
"Pilot '" + slot.uuid + "' reserved for VM " +
slot.vmid + " @ host '" + hostname + "'. Started at: " +
localFormat.format(timestamp.getTime()) + ". VM " +
"running time ends at: " +
localFormat.format(stop.getTime()) + ". Slot will " +
"end itself at approximately: " +
localFormat.format(slotstop.getTime());
if (lager.eventLog) {
logger.info(msg);
} else {
logger.debug(msg);
}
this.schedulerAdapter.slotReserved(slot.vmid,
timestamp,
stop,
hostname);
} catch (ManageException e) {
if (logger.isDebugEnabled()) {
logger.error(e.getMessage(), e);
} else {
logger.error(e.getMessage());
}
} catch (SlotNotFoundException e) {
String msg = "Severe problem, hearing about a slot being " +
"reserved but service has no record of it. Slotid: " +
slotid + ", hostname: " + hostname + " (can't qdel or " +
"cancel it, we don't know the LRM handle or workspace ID)";
logger.error(msg);
}
}
/**
* The pilot reports it started running but the slot was not successfully
* reserved beacuse of some problem.
*
* @param slotid uuid
* @param hostname hostname
* @param error error message
*/
public void errorReserving(String slotid, String hostname, String error) {
try {
PilotSlot slot = this.getSlotAndAssignVM(slotid, hostname);
String id = "Pilot '" + slotid + "'";
if (hostname != null) {
id += " @ host '" + hostname + "'";
}
if (slot.terminal) {
logger.error(id + " had an error " +
"reserving: '" + error + "', nothing to do " +
"slot was already terminal -- unexpected this " +
"would be the case because this is the first " +
"we've heard from slot/sub-slot (?)");
} else {
logger.error(id + " had an error reserving: '" + error +
"' (cancelling vm #" + slot.vmid + " without " +
"running trash)");
// this eventually causes this.releaseSpace() to be called
// unless there was a race
this.cancelWorkspaceNoTrash(slot.vmid, new Exception(error));
this.db.setSlotTerminal(slot);
}
} catch (WorkspaceDatabaseException e) {
logger.error(
getDBError("problem reserving",
slotid,
hostname,
e.getMessage()),
e);
} catch (SlotNotFoundException e) {
logger.error(getNoSlotError("problem reserving", slotid, hostname));
}
}
/**
*
* The pilot reports that it has been interrupted and has determined the
* signal was unexpected. This can happen in three situations:
*
* 1. The LRM or administrator has decided to preempt the pilot for
* whatever reason.
*
* 2. The node has been rebooted or shutdown.
*
* 3. The LRM job was cancelled by the slot manager (this class).
*
* In each situation the pilot attempts to wait a specific (configurable)
* ratio of the provided grace period. In cases #1 and #2 this gives the
* slot manager time to handle the problem (currently this involves running
* shutdown-trash on all VMs in the slot). In case #3 the slot manager can
* just ignore this notification since it is already done with the slot
* (which is why it cancelled the LRM job).
*
* @param slotid uuid
* @param hostname hostname
* @param timestamp the time that pilot sent this (second resolution only)
* used to compute if we should act on it
*/
public void earlyUnreserving(String slotid,
String hostname,
Calendar timestamp) {
try {
// SlotNotFoundException expected if we called qdel
// (which is the usual situation)
PilotSlot slot = this.db.getSlot(slotid, hostname);
StringBuffer buf = new StringBuffer();
buf.append("Pilot '")
.append(slotid);
if (hostname != null) {
buf.append("' @ host '")
.append(hostname);
}
buf.append("' is being preempted early. Cancelling vm #")
.append(slot.vmid);
// Just before the notification was sent
long timestampLong = timestamp.getTimeInMillis();
// Now, which could be at any arbitrary time.
Calendar now = Calendar.getInstance();
long nowLong = Calendar.getInstance().getTimeInMillis();
long difference = nowLong - timestampLong;
difference = difference / 1000; //convert to seconds
// Because this is an unusual situation, do a lot of logging
localFormat.setCalendar(timestamp);
int timestampGMTOffset = timestamp.getTimeZone().getRawOffset();
buf.append(" || Notification sent @ ")
.append(localFormat.format(timestamp.getTime()))
.append(" -- GMT offset ")
.append(timestampGMTOffset);
localFormat.setCalendar(now);
int nowGMTOffset = now.getTimeZone().getRawOffset();
buf.append(" || Now: ")
.append(localFormat.format(now.getTime()))
.append(" -- GMT offset ")
.append(nowGMTOffset);
// This check is mostly here if the service was down and the
// notification consumer is only now getting to the notifications.
// We allow for a certain amount of inaccuracy on top of the grace
// period (e.g. only using second resolution on the timestamp).
int horizon = this.grace + 2;
boolean trash = true;
if (difference > horizon) {
trash = false;
}
buf.append(" || Difference in seconds is ")
.append(difference)
.append(" which means we will ");
if (!trash) {
buf.append("not ");
}
buf.append("run shutdown-trash now (difference ");
if (trash) {
buf.append(" < ");
} else {
buf.append(" > ");
}
buf.append("than now + ")
.append(horizon)
.append(" seconds.");
final String msg = buf.toString();
if (lager.eventLog) {
logger.info(Lager.ev(slot.vmid) + msg);
} else {
logger.debug(Lager.ev(slot.vmid) + msg);
}
final Exception e = new Exception("early LRMS preemption");
if (trash) {
this.cancelWorkspace(slot.vmid, e);
} else {
this.cancelWorkspaceNoTrash(slot.vmid, e);
}
this.db.setSlotTerminal(slot);
} catch (WorkspaceDatabaseException e) {
final String msg = getDBError("starting early-unreserving",
slotid,
hostname,
e.getMessage());
logger.error(msg, e);
} catch (SlotNotFoundException e) {
String id = "'" + slotid + "'";
if (hostname != null) {
id += " @ host '" + hostname + "'";
}
logger.debug("Pilot " + id + " is being preempted and " +
"we do not have a record of this slot anymore. This " +
"is the expected situation if we have called qdel (this " +
"could also have happened if the service just " +
"recovered from being down).");
}
}
/**
* The pilot reports that there was a problem early unreserving, there is
* no action to take. An error message will usually accompany this (for
* logging to service logs).
*
* @param slotid uuid
* @param hostname hostname
* @param error error message
*/
public void errorEarlyUnreserving(String slotid,
String hostname,
String error) {
logger.error("Pilot '" + slotid + "' had an error " +
"early-unreserving: '" + error);
try {
// SlotNotFoundException expected
PilotSlot slot = this.db.getSlot(slotid, hostname);
String id = "Pilot '" + slotid + "'";
if (hostname != null) {
id += " @ host '" + hostname + "'";
}
logger.warn(id + " had an error early-unreserving. Antecedent " +
"unreserving notification either never have made " +
"it or service's clean up crossed paths with the " +
"pilot's grace period expiring" +
": (cancelling vm " + slot.vmid + "without running " +
"trash, a resource-not-found error is likely)");
// this eventually causes this.releaseSpace() to be called
// unless there was a race
final Exception e = new Exception("LRMS preemption with error");
this.cancelWorkspaceNoTrash(slot.vmid, e);
this.db.setSlotTerminal(slot);
} catch (WorkspaceDatabaseException e) {
final String msg = getDBError("problem early-unreserving",
slotid,
hostname,
e.getMessage());
logger.error(msg, e);
} catch (SlotNotFoundException e) {
// expected
}
}
/**
* The pilot reports that it has begun unreserving the slot, there is
* nothing to be done now, this is the end (whether it passes or fails). If
* there was something the manager was expected to do, earlyUnreserving
* would have been called.
*
* @param slotid uuid
* @param hostname hostname
*/
public void unreserving(String slotid, String hostname) {
try {
PilotSlot slot = this.db.getSlot(slotid, hostname);
String id = "Pilot '" + slotid + "'";
if (hostname != null) {
id += " @ host '" + hostname + "'";
}
logger.error(id + " is being unreserved: " +
"Cancelling vm #" + slot.vmid +
" (which should be gone already).");
// this eventually causes this.releaseSpace() to be called
// unless there was a race
this.cancelWorkspaceNoTrash(slot.vmid, null);
this.db.setSlotTerminal(slot);
} catch (WorkspaceDatabaseException e) {
String msg = getDBError("starting unreserving",
slotid,
hostname,
e.getMessage());
logger.error(msg, e);
} catch (SlotNotFoundException e) {
String id = "Pilot '" + slotid + "'";
if (hostname != null) {
id += " @ host '" + hostname + "'";
}
logger.debug(id + " is shutting down and " +
"we do not have a record of this slot anymore. This " +
"is the expected situation.");
}
}
/**
* The pilot reports it has killed VMs.
*
* @param slotid uuid
* @param hostname hostname
* @param killed array of killed VM IDs
*/
public void kills(String slotid, String hostname, String[] killed) {
if (killed == null) {
logger.error("erroneous notification, killed but null VM list");
return;
}
if (killed.length == 0) {
logger.error("erroneous notification, killed but empty VM list");
return;
}
StringBuffer buf = new StringBuffer();
buf.append("'")
.append(killed[0])
.append("'");
for (int i = 1; i < killed.length; i++) {
buf.append(", '")
.append(killed[i])
.append("'");
}
String id = "Pilot '" + slotid + "'";
if (hostname != null) {
id += " @ host '" + hostname + "'";
}
logger.error(id + "' had to kill these VMS: " + buf.toString());
try {
this.db.setSlotTerminal(this.db.getSlot(slotid, hostname));
} catch (WorkspaceDatabaseException e) {
logger.error(e.getMessage(), e);
} catch (SlotNotFoundException e) {
logger.error(getNoSlotRaceError("killing", slotid, hostname), e);
}
for (int i = 0; i < killed.length; i++) {
// todo: String -> id implementation should be discovered
final int vmid = XenUtil.xenNameToId(killed[i]);
if (vmid < 0) {
logger.error("We don't know about this killed VM '" +
killed[i] + "', nothing to do.");
} else {
final Exception e =
new Exception("LRMS preemption with error (kills)");
this.cancelWorkspaceNoTrash(vmid, e);
}
}
}
/**
* The pilot reports that it can not successfully unreserve the slot, this
* is no action to take. An error message will usually accompany this (for
* logging to service logs).
*
* @param slotid uuid
* @param hostname hostname
* @param error error message
*/
public void errorUnreserving(String slotid, String hostname, String error) {
logger.error("Pilot '" + slotid + "' had an error " +
"unreserving: '" + error);
try {
// SlotNotFoundException expected
PilotSlot slot = this.db.getSlot(slotid, hostname);
String id = "Pilot '" + slotid + "'";
if (hostname != null) {
id += " @ host '" + hostname + "'";
}
logger.error(id + " had an error " +
"unreserving but the previously expected " +
"unreserving notification seems to never have made " +
"it: (cancelling vm without running trash: " +
slot.vmid + ")");
// this eventually causes this.releaseSpace() to be called
// unless there was a race
this.cancelWorkspaceNoTrash(slot.vmid, new Exception(error));
this.db.setSlotTerminal(slot);
} catch (WorkspaceDatabaseException e) {
String msg = getDBError("problem unreserving",
slotid,
hostname,
e.getMessage());
logger.error(msg, e);
} catch (SlotNotFoundException e) {
// expected
}
}
private static String getDBError(String state,
String slotid,
String hostname,
String err) {
return "Severe problem, hearing about a pilot " +
state + " but service has a DB problem. " +
"Slot: " + slotid + ", hostname: " + hostname +
" (would have cancelled workspace" +
" instance if we knew what it was). DB problem: " + err;
}
private static String getNoSlotError(String state,
String slotid,
String hostname) {
return "Problem, hearing about a pilot " +
state + " but service has no record of it. " +
"Slotid: " + slotid + ", hostname: " + hostname +
" (would have cancelled workspace" +
" instance if we knew what it was)";
}
private static String getNoSlotRaceError(String state,
String slotid,
String hostname) {
return "Probably OK race condition because of external, " +
"asynchronous notifications. Hearing about a pilot " +
state + " but service has no record of it. Slotid: " +
slotid + ", hostname = " + hostname;
}
// see reserved() and errorReserving()
PilotSlot getSlotAndAssignVM(String uuid, String hostname)
throws WorkspaceDatabaseException, SlotNotFoundException {
synchronized (this.groupLock) {
return this.db.getSlotAndAssignVMImpl(uuid, hostname);
}
}
/* ******************** */
/* Service interactions */
/* ******************** */
private void cancelWorkspace(int vmid, Exception e) {
this.cancelWorkspace(vmid, true, e);
}
private void cancelWorkspaceNoTrash(int vmid, Exception e) {
this.cancelWorkspace(vmid, false, e);
}
private void cancelWorkspace(int vmid,
boolean trash,
Throwable t) {
final InstanceResource resource;
try {
resource = this.instHome.find(vmid);
} catch (Exception e) {
final String msg = "Couldn't find workspace " + Lager.id(vmid) +
" to cancel, already gone.";
logger.error(msg);
return;
}
if (!trash) {
resource.setVMMaccessOK(false);
}
try {
final int curr = resource.getState();
// no need to set a new state if it is already corrupted
// (this is a check then act problem)
if (curr < WorkspaceConstants.STATE_CORRUPTED) {
resource.setState(curr + WorkspaceConstants.STATE_CORRUPTED,
null);
}
} catch (ManageException e) {
final String msg = "Couldn't set corrupted state on workspace "
+ Lager.id(vmid) +
", already gone.";
if (logger.isDebugEnabled()) {
logger.error(msg, e);
} else {
logger.error(msg);
}
}
}
public ResourcepoolEntry addNode(String hostname, String pool, String networks, int memory,
boolean active)
throws NodeExistsException, NodeManagementDisabled {
throw new NodeManagementDisabled(REMOTE_NODE_MGR_DISABLED);
}
public List<ResourcepoolEntry> getNodes() throws NodeManagementDisabled {
throw new NodeManagementDisabled(REMOTE_NODE_MGR_DISABLED);
}
public ResourcepoolEntry getNode(String hostname) throws NodeManagementDisabled {
throw new NodeManagementDisabled(REMOTE_NODE_MGR_DISABLED);
}
public ResourcepoolEntry updateNode(String hostname, String pool, String networks,
Integer memory, Boolean active)
throws NodeInUseException, NodeNotFoundException, NodeManagementDisabled {
throw new NodeManagementDisabled(REMOTE_NODE_MGR_DISABLED);
}
public boolean removeNode(String hostname)
throws NodeInUseException, NodeManagementDisabled {
throw new NodeManagementDisabled(REMOTE_NODE_MGR_DISABLED);
}
public String getVMMReport() {
return "No VMM report when pilot is configured.";
}
public String getAccountString(String userDN, String accountingType) {
String accountString = null;
if (accountingType == null) {
accountString = null;
}
else if (accountingType.equalsIgnoreCase("dn")) {
accountString = userDN;
}
else if (accountingType.equalsIgnoreCase("displayname")) {
try {
String userID = authzDBAdapter.getCanonicalUserIdFromDn(userDN);
final List<UserAlias> aliasList = authzDBAdapter.getUserAliases(userID);
for (UserAlias alias : aliasList) {
if (alias.getAliasType() == AuthzDBAdapter.ALIAS_TYPE_DN) {
accountString = alias.getFriendlyName();
}
}
logger.error("Can't find display name for '" + userDN + "'. "
+ "No accounting string will be sent to PBS.");
}
catch (Exception e) {
logger.error("Can't connect to authzdb db. No accounting string will be sent to PBS.");
}
}
else if (accountingType.equalsIgnoreCase("group")) {
try {
GroupAuthz groupAuthz = (GroupAuthz)this.authzCallout;
accountString = groupAuthz.getGroupName(userDN);
}
catch (Exception e) {
logger.error("Problem getting group string. Are you sure you're using Group or SQL authz?");
logger.debug("full error: " + e);
}
}
else {
logger.error("'" + accountingType + "' isn't a valid accounting string type. "
+ "No accounting string will be sent to PBS.");
}
return accountString;
}
}