package edu.berkeley.chukwa_xtrace;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.File;
import java.util.*;
import org.apache.hadoop.io.Text;
import edu.berkeley.chukwa_xtrace.XtrExtract.PtrReverse;
import edu.berkeley.xtrace.reporting.Report;
/**
* Encapsulates a causal graph; nodes are xtrace reports.
*
*/
public class CausalGraph implements Iterable<Report> {
/**
* Returns the distance from report src to dest.
* Should be positive if dest happened after src
* @author asrabkin
*
*/
public static class RDistMetric {
long dist(Report src, Report dest) {
return getTS(src);
}
}
public static class IgnoreReportsMetric extends RDistMetric {
List<Report> pathToIgnore;
/**
* Path should be reverse-ordered, same as longestPath returns
* @param pathToIgnore
*/
public IgnoreReportsMetric(List<Report> pathToIgnore) {
this.pathToIgnore = pathToIgnore;
}
@Override
long dist(Report src, Report dest) {
for(int i =0; i < pathToIgnore.size()-1; ++i) {
if((pathToIgnore.get(i+1) == src) && (pathToIgnore.get(i) == dest))
return 0;
}
return getTS(src);
}
}
private Map<String, Report> reports;
private Report start;
private Report end;
public CausalGraph(Map<String, Report> reports) {
this.reports = reports;
}
public CausalGraph() {
reports = new LinkedHashMap<String, Report>();
}
public Report getStart() {
return start;
}
public void setStart(Report start) {
this.start = start;
}
public Report getEnd() {
return end;
}
public void setEnd(Report end) {
this.end = end;
}
public void add(Report r) {
String opID = r.getMetadata().getOpIdString();
reports.put(opID, r);
}
/////// Graph-analytic functions
public Set<Report> predecessors(Report p) {
HashSet<Report> predecessors = new HashSet<Report>();
Queue<Report> bfsQ = new LinkedList<Report>();
bfsQ.add(p);
while(!bfsQ.isEmpty()) {
Report r = bfsQ.remove();
assert r!= null;
predecessors.add(r);
List<String> backEdges = r.get("Edge");
if(backEdges != null)
for(String pred:backEdges) {
Report pre = reports.get(pred);
if(pre != null)
bfsQ.add(pre);
}
}
return predecessors;
}
public List<Report> topoSort(PtrReverse reverser) {
HashMap<String, Integer> counts = new HashMap<String, Integer>();
Queue<Report> zeroInlinkReports = new LinkedList<Report>();
//FIXME: could usefully compare reports.size() with numReports;
//that would measure duplicate reports
//increment link counts for children
for(Report r: reports.values()){
String myOpID = r.getMetadata().getOpIdString();
int parentCount = reverser.setupForwardPointers(reports, r, myOpID);
//if there weren't any parents, we can dequeue
if(parentCount == 0)
zeroInlinkReports.add(r);
else
counts.put(myOpID, parentCount);
}
//at this point, we have a map from metadata to report, and also
//from report op ID to inlink count.
//next step is to do a topological sort.
ArrayList<Report> finalOutput = new ArrayList<Report>();
while(!zeroInlinkReports.isEmpty()) {
Report r = zeroInlinkReports.remove();
List<String> outLinks = r.get(XtrExtract.OUTLINK_FIELD);
if(outLinks != null) {
for(String outLink: outLinks) {
Integer oldCount = counts.get(outLink);
if(oldCount == null) {
oldCount = 0; //FIXME: can this happen?
//Means we have a forward-edge to a node which we haven't ever set up a link count for
// log.warn(taskIDString+": found an in-edge where none was expected");
} if(oldCount == 1) {
zeroInlinkReports.add(reports.get(outLink));
}
counts.put(outLink, oldCount -1);
}
}
}
return finalOutput;
}
/////// Performance-analytic functions
private static final long getTS(Report r) {
List<String> staTL = r.get("Timestamp");
if(staTL != null && staTL.size() > 0) {
double t = Double.parseDouble(staTL.get(0));
return Math.round(1000 * t);
}
return Long.MIN_VALUE;
}
/**
* Returns the longest path ending at endID
*
* Path is in reversed order, starting with endID and going forwards.
*
* @param endID
* @return
*/
public List<Report> longestPath(String endID) {
return longestPath(new RDistMetric(), endID);
}
public List<Report> longestPath(RDistMetric metric, String endID) {
//if we have the reports in topological order, this should be easy.
//Just take max of all predecessors seen until that point.
//alternatively, could start at the end and walk backwards
ArrayList<Report> backpath = new ArrayList<Report>();
Report cur = reports.get(endID);
do {
backpath.add(cur);
Report limitingPred = null;
long latestPrereq = Long.MIN_VALUE;
for(String predID: cur.get("Edge")) {
Report pred = reports.get(predID);
long finishTime = metric.dist(pred, cur);
if( finishTime > latestPrereq) {
latestPrereq = finishTime;
limitingPred = pred;
}
cur = limitingPred;
}
} while(cur != null && cur.get("Edge") != null);
//should be able to just walk forward, keeping trac
return backpath;
}
/**
* Expect path to be sorted backwards.
* @param path
* @return
*/
public static long onHostTimes(List<Report> path) {
long time =0;
for(int i =0; i < path.size()-1; ++i) {
Report src = path.get(i+1);
Report dest = path.get(i);
List<String> srcHost = src.get("Host"), destHost = dest.get("Host");
if(srcHost != null && srcHost.size() > 0 && destHost != null && destHost.size() > 0) {
if(srcHost.get(0).equals(destHost.get(0))){
long src_ts = getTS(src);
long dest_ts = getTS(dest);
time += (dest_ts - src_ts);
System.out.println("adding segment of length " + (dest_ts - src_ts));
}
}
}
return time;
}
//// Glue to make CausalGraph look like a pseudocollection
public Iterator<Report> iterator() {
return reports.values().iterator();
}
public int size() {
return reports.size();
}
public Collection<Report> getReports() {
return reports.values();
}
//////IO utils
public void slurpTextFile(File f) throws IOException {
BufferedReader br = new BufferedReader(new FileReader(f));
Report rep;
while((rep = getNextReportFromReader(br)) != null ) {
add(rep);
}
}
private Report getNextReportFromReader(BufferedReader br) throws IOException {
StringBuilder sb = new StringBuilder();
String s;
while((s = br.readLine()) != null ) {
if(s.length() > 1) {
sb.append(s);
sb.append("\n");
} else //stop on blank line, if it isn't the first one we see
if(sb.length() > 1)
break;
}
if(sb.length() < 1)
return null;
return Report.createFromString(sb.toString());
}
}