/*
* Copyright 2011 PA Consulting Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.prodeagle.java.servlets;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.simple.JSONObject;
import com.google.appengine.api.users.UserService;
import com.google.appengine.api.users.UserServiceFactory;
import com.prodeagle.java.Authentication;
import com.prodeagle.java.MemCacheManager;
import com.prodeagle.java.counters.CounterNamesManager;
import com.prodeagle.java.counters.CounterUtil;
public class HarvestHandler extends HttpServlet {
/**
* Auto-generated for Serializable
*/
private static final long serialVersionUID = 7461496095351578024L;
private static final Logger _logger = Logger.getLogger(HarvestHandler.class.getName());
private static final UserService userService = UserServiceFactory.getUserService();
private static final String PRODUCTION_CALL = "production_call";
private static final int EXPECTED_MEMCACHE_SERVERS = 1024;
private static final int MAX_LOOK_BACK = 1000 * 60 * 60; //1000 milliseconds * 60 seconds * 60 minutes (1 hour)
public void doGet(HttpServletRequest req, HttpServletResponse resp) {
//check for the existence of ?viewer=xyz or ?administrator=xyz
if (null != req.getParameter("viewer") || null != req.getParameter("administrator")) {
_logger.info("Adding user");
Authentication.addUser(req, resp);
} else if (Authentication.isProdEagle(req, resp) || Authentication.isAdministrator(req, resp)) { //check prodeagle first, because admin does some redirecting
_logger.info("Creating report");
createReport(req, resp);
}
}
public void doPost(HttpServletRequest req, HttpServletResponse resp) {
_logger.info(req.getParameterMap().toString());
doGet(req, resp);
}
private Boolean isProductionCall(HttpServletRequest req) {
String productionCallString = req.getParameter(PRODUCTION_CALL);
Boolean isProductionCall = false;
if (null != productionCallString && productionCallString.equalsIgnoreCase("1")) {
isProductionCall = true;
}
_logger.info("Is production call? " + isProductionCall);
return isProductionCall;
}
private long getLastSlot(HttpServletRequest req) {
String lastSlotString = req.getParameter("last_time");
Long lastSlot;
if (null == lastSlotString) {
_logger.info("Last slot not in parameters");
long time = new Date().getTime() - MAX_LOOK_BACK;
lastSlot = CounterUtil.getEpochRounded(new Date(time), MAX_LOOK_BACK);
} else {
_logger.info("Last slot in parameters");
lastSlot = Long.parseLong(lastSlotString) * 1000; //turn it into milliseconds (because prodeagle gives it in seconds)
}
_logger.info("Last slot: " + lastSlot + " (or as a date: " + new Date(lastSlot).toString() + ")");
return lastSlot;
}
@SuppressWarnings("unchecked")
private void createReport(HttpServletRequest req, HttpServletResponse resp) {
Boolean isProductionCall = isProductionCall(req);
long slot = CounterUtil.getEpochRounded();
JSONObject result = initaliseDefaultResult(slot);
CounterNamesManager cnm = CounterUtil.getDefaultCounterNamesManager();
Boolean allDataInaccurate = wasDataLostSinceLastHarvest(CounterUtil.NAMESPACE, slot, isProductionCall);
Set<String> allCounterNames = cnm.allCounterNames();
Set<String> updateKeys = new HashSet<String>();
Set<Object[]> updates = new HashSet<Object[]>();
long lastSlot = getLastSlot(req);
while (slot >= lastSlot) {
long gap = new Date().getTime();
Map<String, Object> slotUpdates = MemCacheManager.getMultipleCounters(allCounterNames, String.valueOf(slot), CounterUtil.NAMESPACE);
if (isProductionCall) {
MemCacheManager.deleteMulti(slotUpdates.keySet(), CounterUtil.NAMESPACE);
}
result.put("ms_of_data_lost", computeMsOfDataLost(gap, (Long) result.get("ms_of_data_lost")));
updates.add(new Object[]{ slot, slotUpdates });
updateKeys.addAll(slotUpdates.keySet());
slot -= CounterUtil.MIN_SLOT_SIZE;
}
if (!allDataInaccurate) {
//have we lost any data since we first checked?
allDataInaccurate = wasDataLostSinceLastHarvest(CounterUtil.NAMESPACE, slot);
}
_logger.info("All data inaccurate? " + allDataInaccurate);
//store the result of wasDataLost
result.put("all_data_inaccurate", allDataInaccurate);
for (String updateKey : updateKeys) {
JSONObject counters = (JSONObject) result.get("counters");
updateKey = updateKey.substring(updateKey.indexOf("_") + 1, updateKey.length());
if (!counters.containsKey(updateKey)) {
counters.put(updateKey, new JSONObject());
}
for (Object[] update : updates) {
Long _slot = (Long) update[0];
Map<String, Long> _slotUpdates = (Map<String, Long>) update[1];
try {
Object delta = _slotUpdates.get(_slot + "_" + updateKey);
if (null != delta) {
JSONObject x = (JSONObject) counters.get(updateKey);
x.put(_slot / 1000, delta); //prodeagle time stamps are in seconds, not milliseconds
counters.put(updateKey, x);
}
} catch (Exception e) {
_logger.info("Test: " + e);
}
}
}
/*
* Write the response out. If the request is for json / production
* then write out the JSON string without anything else.
*
* If not json/production, write out a helpful list of the counters
* and their values
*/
try {
if (isProductionCall || null != req.getParameter("json")) {
resp.setContentType("text/plain; charset=utf-8");
String jsonString = result.toJSONString();
resp.getWriter().print(jsonString);
} else {
resp.setContentType("text/html");
PrintWriter writer = resp.getWriter();
writer.print("<html><head><title>ProEagle Stats</title><style> td { padding: 5px }</style></head><body>");
writer.print(String.format("<h3>Data since last export: %s UTC</h3>", new Date()));
writer.print("<a href='http://www.prodeagle.com'>Go to ProdEagle dashboard</a>");
writer.print(String.format("<br /><br /><a href='%s'>Logout</a>",
userService.createLogoutURL(req.getRequestURI())));
JSONObject counters = (JSONObject) result.get("counters");
writer.print("<table><tr><td>Number of counters:</td><td>" + counters.size() + "</td></tr></table>");
List<String> sortedKeys = new ArrayList<String>(counters.keySet());
Collections.sort(sortedKeys);
for (Object counterKey : sortedKeys) {
String counterName = (String) counterKey;
JSONObject json = (JSONObject) counters.get(counterName);
if (!json.isEmpty()) {
writer.print("<table><thead><th colspan=\"3\">" + counterName + "</th></thead><tbody>");
for (Object key : json.keySet()) {
writer.print("<tr><td>");
writer.print(key);
writer.print("</td><td style=\"border-left: 1px solid grey; border-right: 1px solid grey;\">");
writer.print(new Date((Long) key * 1000).toString()); //modify the dates back to include milliseconds
writer.print("</td><td>");
writer.print(json.get(key));
writer.print("</td></tr>");
}
writer.print("</tbody></table>");
}
}
writer.print("</body></html>");
writer.flush();
}
} catch (IOException e) {
_logger.severe("Failure to write response: " + e);
}
}
/**
*
* @param gap -
* @param currentMsOfDataLost
* @return
*/
private long computeMsOfDataLost(long gap, long currentMsOfDataLost) {
long thisGap = (new Date().getTime() - gap);
long max = Math.max(thisGap, currentMsOfDataLost);
return max;
}
private Boolean wasDataLostSinceLastHarvest(String namespace, long slot) {
return wasDataLostSinceLastHarvest(namespace, slot, false);
}
private Boolean wasDataLostSinceLastHarvest(String namespace, long slot, Boolean isProductionCall) {
List<String> lostDataCheckKeys = new ArrayList<String>();
int i = 0;
while (i < EXPECTED_MEMCACHE_SERVERS) {
lostDataCheckKeys.add("last_slot_" + i);
i++;
}
Map<String, Object> lostDataCheck = MemCacheManager.getMultipleCounters(lostDataCheckKeys, CounterUtil.NAMESPACE);
if (isProductionCall) {
MemCacheManager.deleteMulti(lostDataCheckKeys, CounterUtil.NAMESPACE);
Map<String, Long> nextLostData = new HashMap<String, Long>();
for (String key : lostDataCheckKeys) {
nextLostData.put(key, 1L);
}
MemCacheManager.storeMultipleCounters(nextLostData, CounterUtil.NAMESPACE, 0L);
}
//check if the
if (lostDataCheck.values().size() != lostDataCheckKeys.size()) {
_logger.warning("ProdEagle counters lost before " + slot);
return true;
} else {
return false;
}
}
@SuppressWarnings("unchecked")
/**
* Creates a default result, with no results yet filled
* @param slot - the epoch time
* @return - a JSON Object with time, counters, ms_of_data_lost and version all set
*/
private JSONObject initaliseDefaultResult(long slot) {
JSONObject json = new JSONObject();
json.put("time", slot / 1000); //turn into a python time (i.e. without milliseconds)
json.put("counters", new JSONObject());
json.put("ms_of_data_lost", 0L);
json.put("version", 1.0);
return json;
}
}