/**
* Copyright 2012 Comcast Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.comcast.cns.controller;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.log4j.Logger;
import org.w3c.dom.Element;
import com.comcast.cmb.common.controller.AdminServletBase;
import com.comcast.cmb.common.controller.CMBControllerServlet;
import com.comcast.cmb.common.model.User;
import com.comcast.cmb.common.persistence.IUserPersistence;
import com.comcast.cmb.common.persistence.PersistenceFactory;
import com.comcast.cmb.common.util.CMBProperties;
import com.comcast.cmb.common.util.PersistenceException;
import com.comcast.cmb.common.util.XmlUtil;
/**
* Subscriptions admin page
* @author bwolf
*
*/
public class CNSWorkerStatePageServlet extends AdminServletBase {
private static final long serialVersionUID = 1L;
private static Logger logger = Logger.getLogger(CNSWorkerStatePageServlet.class);
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (redirectNonAdminUser(request, response)) {
return;
}
CMBControllerServlet.valueAccumulator.initializeAllCounters();
response.setContentType("text/html");
PrintWriter out = response.getWriter();
Map<?, ?> parameters = request.getParameterMap();
//String userId = request.getParameter("userId");
//connect(userId);
IUserPersistence userHandler = PersistenceFactory.getUserPersistence();
User cnsAdminUser;
try {
cnsAdminUser = userHandler.getUserByName(CMBProperties.getInstance().getCNSUserName());
} catch (PersistenceException ex) {
throw new ServletException(ex);
}
String currentDataCenter = null;
if (parameters.containsKey("ClearWorkerQueues")) {
try {
String host = request.getParameter("Host");
httpGet(cnsServiceBaseUrl + "?Action=ManageService&Host="+host+"&Task=ClearWorkerQueues&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_clear_queues", ex);
throw new ServletException(ex);
}
} else if (parameters.containsKey("RemoveWorkerRecord")) {
try {
String host = request.getParameter("Host");
httpGet(cnsServiceBaseUrl + "?Action=ManageService&Host="+host+"&Task=RemoveWorkerRecord&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_clear_queues", ex);
throw new ServletException(ex);
}
} else if (parameters.containsKey("ClearAPIStats")) {
try {
String url = request.getParameter("Url");
httpGet(url + "?Action=ManageService&Task=ClearAPIStats&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_clear_queues", ex);
throw new ServletException(ex);
}
} else if (parameters.containsKey("RemoveRecord")) {
try {
String host = request.getParameter("Host");
httpGet(cnsServiceBaseUrl + "?Action=ManageService&Host="+host+"&Task=RemoveRecord&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_clear_queues", ex);
throw new ServletException(ex);
}
} else if (parameters.containsKey("StartWorker") ||parameters.containsKey("StopWorker")) {
try {
String dataCenter = request.getParameter("DataCenter");
String actionString;
if(parameters.containsKey("StartWorker")){
actionString = "StartWorker";
} else {
actionString = "StopWorker";
}
httpGet(cnsServiceBaseUrl + "?Action=ManageService&DataCenter="+dataCenter+"&Task="+actionString+"&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_start_or_stop_worker ", ex);
throw new ServletException(ex);
}
} else if(parameters.containsKey("currentDataCenter")){
currentDataCenter = request.getParameter("currentDataCenter");
}
if(currentDataCenter == null){
currentDataCenter = CMBProperties.getInstance().getCMBDataCenter();
}
out.println("<html>");
this.header(request, out, "CNS Worker State");
out.println("<body>");
//show drop down box for data center at the top
String url = null;
try{
url = cnsServiceBaseUrl + "?Action=GetAPIStats&SubTask=GetDataCenter&AWSAccessKeyId=" + cnsAdminUser.getAccessKey();
String workerStateXml = httpGet(url);
Element root = XmlUtil.buildDoc(workerStateXml);
List<Element> DataCenterList = XmlUtil.getCurrentLevelChildNodes(XmlUtil.getCurrentLevelChildNodes(root, "GetAPIStatsResult").get(0), "DataCenter");
out.println("<table cellspacing='0' cellpadding='0'><tr>");
out.println("<td valign='top'>Data Center: </td>");
out.println("<td><div>");
out.print("<form action=\"\" method=\"POST\">");
out.println("<select class=\"service-area\" name='currentDataCenter' onchange='this.form.submit()'>");
String dataCenter = null;
for(Element dataCenterElement:DataCenterList){
dataCenter=dataCenterElement.getFirstChild().getNodeValue();
if(dataCenter.equals(currentDataCenter)){
out.println("<option selected value=\""+dataCenter+"\">"+dataCenter+"</option>");
}else {
out.println("<option value=\""+dataCenter+"\">"+dataCenter+"</option>");
}
}
out.println("</select>");
out.println("<noscript><input type=\"submit\" value=\"Submit\"></noscript>");
out.println("</form>");
out.println("</div></td></tr></table>");
} catch (Exception ex) {
out.println("<p>Unable to reach " + url + ": "+ex.getMessage()+"</p>");
logger.error("", ex);
}
//show detailed stats
url = null;
try {
url = cnsServiceBaseUrl + "?Action=GetWorkerStats&DataCenter="+currentDataCenter+"&AWSAccessKeyId=" + cnsAdminUser.getAccessKey();
String workerStateXml = httpGet(url);
Element root = XmlUtil.buildDoc(workerStateXml);
List<Element> statsList = XmlUtil.getCurrentLevelChildNodes(XmlUtil.getCurrentLevelChildNodes(root, "GetWorkerStatsResult").get(0), "Stats");
out.println("<h2 align='left'>CNS Worker Stats</h2>");
out.println("<span class='simple'><table border='1'>");
out.println("<tr><th>Host</th><th>Jmx Port</th><th>Mode</th><th>Data Center</th><th>Msg Published</th>");
out.println("<th>Producer Heartbeat</th><th>Active</th><th>Consumer Heartbeat</th><th>Active</th>");
out.println("<th>Delivery Queue Size</th><th>Redelivery Queue Size</th><th>Consumer Overloaded</th><th>Cqs Service Available</th><th>Http Pool Size</th><th></th><th></th></tr>");
String alarmColor = " bgcolor='#C00000'";
String okColor = " bgcolor='#00C000'";
int deliveryQueueMaxSize = CMBProperties.getInstance().getCNSDeliveryHandlerJobQueueLimit();
int redeliveryQueueMaxSize = CMBProperties.getInstance().getCNSReDeliveryHandlerJobQueueLimit();
Map<String, Integer> endpointErrorCounts = new HashMap<String, Integer>();
for (Element stats : statsList) {
out.println("<tr>");
String host = XmlUtil.getCurrentLevelTextValue(stats, "IpAddress");
out.println("<td>"+host+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "JmxPort")+"</td>");
String mode = XmlUtil.getCurrentLevelTextValue(stats, "Mode");
out.println("<td>"+mode+"</td>");
String dataCenter = XmlUtil.getCurrentLevelTextValue(stats, "DataCenter");
out.println("<td>"+dataCenter+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "NumPublishedMessages")+"</td>");
out.println("<td>"+new Date(Long.parseLong(XmlUtil.getCurrentLevelTextValue(stats, "ProducerTimestamp")))+"</td>");
boolean activeProducer = Boolean.parseBoolean(XmlUtil.getCurrentLevelTextValue(stats, "ActiveProducer"));
out.println("<td"+(!activeProducer && mode.contains("Producer") ? alarmColor : okColor)+">"+activeProducer+"</td>");
out.println("<td>"+new Date(Long.parseLong(XmlUtil.getCurrentLevelTextValue(stats, "ConsumerTimestamp")))+"</td>");
boolean activeConsumer = Boolean.parseBoolean(XmlUtil.getCurrentLevelTextValue(stats, "ActiveConsumer"));
out.println("<td"+(!activeConsumer && mode.contains("Consumer") ? alarmColor : okColor)+">"+activeConsumer+"</td>");
int deliveryQueueSize = Integer.parseInt(XmlUtil.getCurrentLevelTextValue(stats, "DeliveryQueueSize"));
out.println("<td"+(1.0*deliveryQueueSize/deliveryQueueMaxSize >= 0.75 ? alarmColor : okColor)+">"+deliveryQueueSize+"</td>");
int redeliveryQueueSize = Integer.parseInt(XmlUtil.getCurrentLevelTextValue(stats, "RedeliveryQueueSize"));
out.println("<td"+(1.0*redeliveryQueueSize/redeliveryQueueMaxSize >= 0.75 ? alarmColor : okColor)+">"+redeliveryQueueSize+"</td>");
boolean consumerOverloaded = Boolean.parseBoolean(XmlUtil.getCurrentLevelTextValue(stats, "ConsumerOverloaded"));
out.println("<td"+(consumerOverloaded ? alarmColor : okColor)+">"+consumerOverloaded+"</td>");
boolean cqsServiceAvailable = Boolean.parseBoolean(XmlUtil.getCurrentLevelTextValue(stats, "CqsServiceAvailable"));
out.println("<td"+((!cqsServiceAvailable && (activeProducer || activeConsumer))? alarmColor : okColor)+">"+cqsServiceAvailable+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "NumPooledHttpConnections")+"</td>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='Host' value='"+host+"'><input type='submit' value='Clear Queues' name='ClearWorkerQueues'/></form></td>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='Host' value='"+host+"'><input type='submit' value='Remove Record' name='RemoveWorkerRecord'/></form></td>");
out.println("</tr>");
List<Element> errorCounts = XmlUtil.getChildNodes(stats, "ErrorCountForEndpoints");
if (errorCounts.size() == 1) {
for (Element errorCount : XmlUtil.getChildNodes(errorCounts.get(0), "Error")) {
String endpoint = XmlUtil.getCurrentLevelAttributeTextValue(errorCount, "endpoint");
Integer count = XmlUtil.getCurrentLevelAttributeIntValue(errorCount, "count");
if (!endpointErrorCounts.containsKey(endpoint)) {
endpointErrorCounts.put(endpoint, count);
} else {
endpointErrorCounts.put(endpoint, endpointErrorCounts.get(endpoint) + count);
}
}
}
}
out.println("</table></span>");
out.println("<table><tr>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='DataCenter' value='"+currentDataCenter+"'><input type='submit' value='Start Workers' name='StartWorker'/></form></td>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='DataCenter' value='"+currentDataCenter+"'><input type='submit' value='Stop Workers' name='StopWorker'/></form></td>");
out.println("</tr></table>");
if (endpointErrorCounts.size() > 0) {
out.println("<h2 align='left'>Failed or timed out Responses during past 60 Seconds by Endpoint</h2>");
out.println("<span class='simple'><table border='1'>");
out.println("<tr><th>Endpoint</th><th>Error Count</th><th><th>");
for (String endpoint : endpointErrorCounts.keySet()) {
out.println("<tr>");
out.println("<td>"+endpoint+"</td>");
out.println("<td>"+endpointErrorCounts.get(endpoint)+"</td>");
int failureSuspensionThreshold = CMBProperties.getInstance().getEndpointFailureCountToSuspensionThreshold();
if (failureSuspensionThreshold != 0 && endpointErrorCounts.get(endpoint) > failureSuspensionThreshold) {
out.println("<td bgcolor='#C00000'><i>suspended</i></td>");
} else {
out.println("<td> </td>");
}
out.println("</tr>");
}
out.println("</table></span>");
}
// api call stats
url = cnsServiceBaseUrl + "?Action=GetAPIStats&DataCenter="+currentDataCenter+"&AWSAccessKeyId=" + cnsAdminUser.getAccessKey();
String apiStateXml = httpGet(url);
root = XmlUtil.buildDoc(apiStateXml);
statsList = XmlUtil.getCurrentLevelChildNodes(XmlUtil.getCurrentLevelChildNodes(root, "GetAPIStatsResult").get(0), "Stats");
out.println("<h2 align='left'>CNS API Stats</h2>");
out.println("<span class='simple'><table border='1'>");
out.println("<tr><th>Ip Address</th><th>Url</th><th>JMX Port</th><th>Data Center</th><th>Time Stamp</th><th>Status</th><th></th><th></th><th></th></tr>");
Map<String, Long> aggregateCallStats = new HashMap<String, Long>();
Map<String, Long> aggregateCallFailureStats = new HashMap<String, Long>();
for (Element stats : statsList) {
out.println("<tr>");
String host = XmlUtil.getCurrentLevelTextValue(stats, "IpAddress");
out.println("<td>" + host + "</td>");
String serviceUrlString = XmlUtil.getCurrentLevelTextValue(stats, "ServiceUrl");
URL serviceUrl = new URL(serviceUrlString);
String endpoint = serviceUrl.getProtocol() + "://" + host + serviceUrl.getPath();
out.println("<td>"+serviceUrlString+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "JmxPort")+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "DataCenter")+"</td>");
out.println("<td>"+new Date(Long.parseLong(XmlUtil.getCurrentLevelTextValue(stats, "Timestamp")))+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "Status")+"</td>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='Url' value='"+endpoint+"'><input type='submit' value='Clear API Stats' name='ClearAPIStats'/></form></td>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='Host' value='"+host+"'><input type='submit' value='Remove Record' name='RemoveRecord'/></form></td>");
out.println("<td><a href='/webui/cmbcallstats'>Stats</a></td>");
out.println("</tr>");
Element callStats = XmlUtil.getChildNodes(stats, "CallStats").get(0);
for (Element action : XmlUtil.getChildNodes(callStats)) {
String actionName = action.getNodeName();
if (!aggregateCallStats.containsKey(actionName)) {
aggregateCallStats.put(actionName, Long.parseLong(XmlUtil.getCurrentLevelTextValue(callStats, actionName)));
} else {
aggregateCallStats.put(actionName, aggregateCallStats.get(actionName) + Long.parseLong(XmlUtil.getCurrentLevelTextValue(callStats, actionName)));
}
}
Element callFailureStats = XmlUtil.getChildNodes(stats, "CallFailureStats").get(0);
for (Element action : XmlUtil.getChildNodes(callFailureStats)) {
String actionName = action.getNodeName();
if (!aggregateCallFailureStats.containsKey(actionName)) {
aggregateCallFailureStats.put(actionName, Long.parseLong(XmlUtil.getCurrentLevelTextValue(callFailureStats, actionName)));
} else {
aggregateCallFailureStats.put(actionName, aggregateCallFailureStats.get(actionName) + Long.parseLong(XmlUtil.getCurrentLevelTextValue(callFailureStats, actionName)));
}
}
}
out.println("</table></span>");
if (aggregateCallStats.keySet().size() > 0) {
out.println("<h2 align='left'>CNS Call Stats</h2>");
out.println("<span class='simple'><table border='1'>");
for (String action : aggregateCallStats.keySet()) {
out.println("<tr><td>"+action+"</td><td>"+aggregateCallStats.get(action)+"</td></tr>");
}
out.println("</table></span>");
}
if (aggregateCallFailureStats.keySet().size() > 0) {
out.println("<h2 align='left'>CNS Call Failure Stats</h2>");
out.println("<span class='simple'><table border='1'>");
for (String action : aggregateCallFailureStats.keySet()) {
out.println("<tr><td>"+action+"</td><td>"+aggregateCallFailureStats.get(action)+"</td></tr>");
}
out.println("</table></span>");
}
} catch (Exception ex) {
out.println("<p>Unable to reach " + url + ": "+ex.getMessage()+"</p>");
logger.error("", ex);
}
out.println("</body></html>");
CMBControllerServlet.valueAccumulator.deleteAllCounters();
}
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
doGet(request, response);
}
}