gr_TimeW_StatesLevel = new RBoxPlot<Pair<Integer,Integer>>("Mutation level,States","Time taken by W",new File("TimeW_statesmutations.pdf")),
gr_TimeRand_StatesLevel = new RBoxPlot<Pair<Integer,Integer>>("Mutation level,States","Time taken by Random",new File("TimeRand_statesmutations.pdf"));
RBagPlot gr_Diff_MutationsToOriginal = new RBagPlot("Mutations/Original Edges","patch size",new File("diff_mutations.pdf")),
gr_Diff_W = new RBagPlot("W f-measure","patch size",new File("diff_w.pdf")),
gr_Diff_MismatchedPairs = new RBagPlot("Mismatched key pairs","Diff/Mutations",new File("diff_pairs.pdf"));
SquareBagPlot gr_F_measures = new SquareBagPlot("W f-measure", "Linear f-measure", new File("f_measure_w_linear.pdf"),0,1,true),
gr_Rand_W = new SquareBagPlot("W f-measure","Rand f-measure",new File("f_measure_rand_W.pdf"),0,1,true);
int approxLimit = 3000;
gr_Diff_MutationsToOriginal.setLimit(approxLimit);
gr_Diff_W.setLimit(approxLimit);
gr_Rand_W.setLimit(approxLimit);
gr_Diff_MismatchedPairs.setLimit(approxLimit);
gr_F_measures.setLimit(approxLimit);
gr_Diff_MutationsToOriginal.setXboundaries(0.,0.5);
gr_Diff_MutationsToOriginal.setYboundaries(1.,1.15);
// Useful values of the threshold
double pairThreshold[] = new double[]{0.2,0.4,0.6,0.8,0.95},lowToHigh[] = new double[]{0.2,0.4,0.6,0.8,0.95};
RBoxPlot<Double> gr_Diff_threshold = new RBoxPlot<Double>("Threshold values, Low to High ratio = 0.5","ratio of patch size to mutations",new File("diff_threshold.pdf")),
gr_Diff_lowtohigh = new RBoxPlot<Double>("Low to High values, threshold ratio = 0.7","ratio of patch size to mutations",new File("diff_lowtohigh.pdf"));
RBoxPlot<Pair<Double,Double>> gr_Diff_thresholdslowhigh = new RBoxPlot<Pair<Double,Double>>("Low to High ratio, Threshold","ratio of patch size to mutations",new File("diff_thresholdlowhigh.pdf"));
RBoxPlot<Double> gr_Diff_k = new RBoxPlot<Double>("attenuation factor","ratio of patch size to mutations",new File("diff_attenuation.pdf"));
for(int graphComplexity=0;graphComplexity < graphComplexityMax;graphComplexity++)
{
int states=initStates+graphComplexity*50;
int alphabet = states/2;
MachineGenerator mg = new MachineGenerator(states, 40, states/10);
int mutationsPerStage = (states/2) / 2;
//System.out.print("\n"+states+": ");
ProgressIndicator progress = new ProgressIndicator(""+states, mutationStages*experimentsPerMutationCategory);
for(int mutationStage = 0;mutationStage<mutationStages;mutationStage++)
{
for(int experiment=0;experiment<experimentsPerMutationCategory;experiment++)
{
ExperimentResult outcome = new ExperimentResult();
while(!outcome.experimentValid)
{
int mutations = mutationsPerStage * (mutationStage+1);
LearnerGraphND origGraph = mg.nextMachine(alphabet, experiment,config,converter);
GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData> mutator = new GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData>(origGraph,r);
mutator.mutate(mutations);
LearnerGraphND origAfterRenaming = new LearnerGraphND(origGraph.config);
Map<CmpVertex,CmpVertex> origToNew = copyStatesAndTransitions(origGraph,origAfterRenaming);
LearnerGraphND mutated = (LearnerGraphND)mutator.getMutated();
Set<Transition> appliedMutations = new HashSet<Transition>();
for(Transition tr:mutator.getDiff())
{
CmpVertex renamedFrom = origToNew.get(tr.getFrom());if (renamedFrom == null) renamedFrom = tr.getFrom();
CmpVertex renamedTo = origToNew.get(tr.getTo());if (renamedTo == null) renamedTo = tr.getTo();
appliedMutations.add(new Transition(renamedFrom,renamedTo,tr.getLabel()));
}
final double perfectLowToHigh=0.7,perfectThreshold=0.5;
// These experiments are only run for the maximal number of states
if (graphComplexity == graphComplexityMax-1)
{
for(double k:new double[]{0,.2,.4,.6,.8,.95})
{
config.setAttenuationK(k);
config.setGdKeyPairThreshold(0.5);
config.setGdLowToHighRatio(perfectLowToHigh);
config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.
linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);
gr_Diff_k.add(k, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
}
for(double threshold:new double[]{0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95})
{
config.setGdKeyPairThreshold(threshold);
config.setGdLowToHighRatio(perfectLowToHigh);
config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.
linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);
gr_Diff_threshold.add(threshold, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
}
for(double lowtohigh:new double[]{0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95})
{
config.setGdKeyPairThreshold(perfectThreshold);
config.setGdLowToHighRatio(lowtohigh);
config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.
linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);
gr_Diff_lowtohigh.add(lowtohigh, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
}
for(int i=0;i<pairThreshold.length;++i)
for(double ratio:lowToHigh)
{
config.setGdKeyPairThreshold(pairThreshold[i]);
config.setGdLowToHighRatio(ratio);
config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.
linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);
gr_Diff_thresholdslowhigh.add(new Pair<Double,Double>(ratio,pairThreshold[i]), outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
}
}
config.setAttenuationK(0.5);
config.setGdKeyPairThreshold(perfectThreshold);
config.setGdLowToHighRatio(perfectLowToHigh);
config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.
linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);
if (!skip)
{
LearnerGraph fromDet = null, toDet = null;
try {
fromDet = mergeAndDeterminize(origAfterRenaming);
toDet = mergeAndDeterminize(mutated);
} catch (IncompatibleStatesException e) {
Helper.throwUnchecked("failed to build a deterministic graph from a nondet one", e);
}
languageDiff(fromDet,toDet,states, graphComplexity,outcome);
}
outcome.experimentValid = true;
progress.next();
gr_Diff_States.add(states, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
gr_W_States.add(states,outcome.getValue(DOUBLE_V.ACCURACY_W));
Pair<Integer,Integer> mutations_states = new Pair<Integer,Integer>(mutationStage+1,states);
Pair<Integer,Integer> states_mutations = new Pair<Integer,Integer>(states,mutationStage+1);
gr_DiffGD_StatesLevel.add(mutations_states, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
gr_DiffW_StatesLevel.add(mutations_states, outcome.getValue(DOUBLE_V.ACCURACY_W));
gr_DiffRand_StatesLevel.add(mutations_states, outcome.getValue(DOUBLE_V.ACCURACY_W));
gr_MutationsToOriginal_StatesLevel.add(mutations_states, outcome.getValue(DOUBLE_V.MUTATIONS_TO_TRANSITIONS));
gr_DiffRand_StatesLevel.add(mutations_states, outcome.getValue(DOUBLE_V.ACCURACY_W));
gr_MismatchedPairs.add(states_mutations,outcome.getValue(DOUBLE_V.MISMATCHED_KEYPAIRS));
gr_Pairs_States.add(states,outcome.getValue(DOUBLE_V.MISMATCHED_KEYPAIRS));
// Time taken
long ns = 1000000L;
gr_TimeDiff_StatesLevel.add(states_mutations, (double)(outcome.getValue(LONG_V.DURATION_GD)/ns));
gr_TimeW_StatesLevel.add(states_mutations, (double)(outcome.getValue(LONG_V.DURATION_W)/ns));
gr_TimeRand_StatesLevel.add(states_mutations, (double)(outcome.getValue(LONG_V.DURATION_RAND)/ns));
gr_TimeDiff_States.add(states,(double)(outcome.getValue(LONG_V.DURATION_GD)/ns));
gr_TimeW_States.add(states,(double)(outcome.getValue(LONG_V.DURATION_W)/ns));
gr_TimeRand_States.add(states,(double)(outcome.getValue(LONG_V.DURATION_RAND)/ns));
gr_Diff_MutationsToOriginal.add(outcome.getValue(DOUBLE_V.MUTATIONS_TO_TRANSITIONS),outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
gr_Diff_W.add(outcome.getValue(DOUBLE_V.ACCURACY_W),outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
gr_Rand_W.add(outcome.getValue(DOUBLE_V.ACCURACY_W),outcome.getValue(DOUBLE_V.ACCURACY_RAND));
gr_Diff_MismatchedPairs.add(outcome.getValue(DOUBLE_V.MISMATCHED_KEYPAIRS),outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));
gr_F_measures.add(outcome.getValue(DOUBLE_V.ACCURACY_W), outcome.getValue(DOUBLE_V.ACCURACY_LINEAR));
gr_FMeasure_Level.add(mutationStage, outcome.getValue(DOUBLE_V.ACCURACY_LINEAR));
}
}
// finished doing this mutation stage, update graphs
gr_Diff_States.drawInteractive(gr);gr_MutationsToOriginal_StatesLevel.drawInteractive(gr);
//gr_Diff_thresholds.drawInteractive(gr);
//gr_Diff_k.drawInteractive(gr);
gr_DiffGD_StatesLevel.drawInteractive(gr);gr_DiffW_StatesLevel.drawInteractive(gr);gr_DiffRand_StatesLevel.drawInteractive(gr);
gr_Diff_MutationsToOriginal.drawInteractive(gr);
gr_W_States.drawInteractive(gr);
gr_Diff_W.drawInteractive(gr);
gr_Rand_W.drawInteractive(gr);
gr_FMeasure_Level.drawInteractive(gr);
//gr_TimeDiff_StatesLevel.drawInteractive(gr);gr_TimeW_StatesLevel.drawInteractive(gr);gr_TimeRand_StatesLevel.drawInteractive(gr);
//gr_MismatchedPairs.drawInteractive(gr);
gr_Pairs_States.drawInteractive(gr);gr_TimeDiff_States.drawInteractive(gr);gr_TimeRand_States.drawInteractive(gr);gr_TimeW_States.drawInteractive(gr);
gr_F_measures.drawInteractive(gr);
//if (mutationStage>0) gr_Diff_MismatchedPairs.drawInteractive(gr);
//ExperimentResult average = getAverage(accuracyStruct,graphComplexity,mutationStage);
//arrayWithDiffResults.add(diffValues);arrayWithKeyPairsResults.add(keyPairsValues);
//arrayWithResultNames.add(""+states+"-"+Math.round(100*average.mutationsToTransitions)+"%");
//System.out.print("["+average+"]");
//List<String> names=arrayWithResultNames.size()>1?arrayWithResultNames:null;
}
}
gr_Diff_States.drawPdf(gr);gr_W_States.drawPdf(gr);gr_MutationsToOriginal_StatesLevel.drawPdf(gr);
gr_Diff_threshold.drawPdf(gr);gr_Diff_lowtohigh.drawPdf(gr);gr_Diff_thresholdslowhigh.drawPdf(gr);gr_Diff_k.drawPdf(gr);
gr_Diff_MutationsToOriginal.drawPdf(gr);
gr_DiffGD_StatesLevel.drawPdf(gr);gr_DiffW_StatesLevel.drawPdf(gr);gr_DiffRand_StatesLevel.drawPdf(gr);
gr_Diff_W.drawPdf(gr);gr_Rand_W.drawPdf(gr);
gr_MismatchedPairs.drawPdf(gr);gr_Diff_MismatchedPairs.drawPdf(gr);
gr_TimeDiff_StatesLevel.drawPdf(gr);gr_TimeW_StatesLevel.drawPdf(gr);gr_TimeRand_StatesLevel.drawPdf(gr);
gr_Pairs_States.drawPdf(gr);gr_TimeDiff_States.drawPdf(gr);gr_TimeRand_States.drawPdf(gr);gr_TimeW_States.drawPdf(gr);
gr_F_measures.drawPdf(gr);