/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.mahout.clustering;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.mahout.clustering.canopy.Canopy;
import org.apache.mahout.clustering.canopy.CanopyDriver;
import org.apache.mahout.clustering.dirichlet.DirichletDriver;
import org.apache.mahout.clustering.dirichlet.UncommonDistributions;
import org.apache.mahout.clustering.dirichlet.models.DistributionDescription;
import org.apache.mahout.clustering.dirichlet.models.GaussianClusterDistribution;
import org.apache.mahout.clustering.evaluation.ClusterEvaluator;
import org.apache.mahout.clustering.evaluation.RepresentativePointsDriver;
import org.apache.mahout.clustering.fuzzykmeans.FuzzyKMeansDriver;
import org.apache.mahout.clustering.kmeans.KMeansDriver;
import org.apache.mahout.clustering.kmeans.TestKmeansClustering;
import org.apache.mahout.clustering.meanshift.MeanShiftCanopyDriver;
import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.Pair;
import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.common.distance.EuclideanDistanceMeasure;
import org.apache.mahout.common.iterator.sequencefile.PathFilters;
import org.apache.mahout.common.iterator.sequencefile.PathType;
import org.apache.mahout.common.iterator.sequencefile.SequenceFileDirIterable;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.VectorWritable;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class TestClusterEvaluator extends MahoutTestCase {
private static final double[][] REFERENCE = { { 1, 1 }, { 2, 1 }, { 1, 2 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 4 }, { 4, 5 },
{ 5, 5 } };
private List<VectorWritable> referenceData = new ArrayList<VectorWritable>();
private final List<VectorWritable> sampleData = new ArrayList<VectorWritable>();
private Map<Integer, List<VectorWritable>> representativePoints;
private List<Cluster> clusters;
private static final Logger log = LoggerFactory.getLogger(TestClusterEvaluator.class);
private Configuration conf;
private FileSystem fs;
private Path testdata;
private Path output;
@Override
@Before
public void setUp() throws Exception {
super.setUp();
conf = new Configuration();
fs = FileSystem.get(conf);
testdata = getTestTempDirPath("testdata");
output = getTestTempDirPath("output");
// Create small reference data set
referenceData = TestKmeansClustering.getPointsWritable(REFERENCE);
// generate larger test data set for the clustering tests to chew on
generateSamples();
}
/**
* Generate random samples and add them to the sampleData
*
* @param num
* int number of samples to generate
* @param mx
* double x-value of the sample mean
* @param my
* double y-value of the sample mean
* @param sd
* double standard deviation of the samples
*/
private void generateSamples(int num, double mx, double my, double sd) {
log.info("Generating {} samples m=[{}, {}] sd={}", new Object[] { num, mx, my, sd });
for (int i = 0; i < num; i++) {
sampleData.add(new VectorWritable(new DenseVector(new double[] { UncommonDistributions.rNorm(mx, sd),
UncommonDistributions.rNorm(my, sd) })));
}
}
private void generateSamples() {
generateSamples(500, 1, 1, 3);
generateSamples(300, 1, 0, 0.5);
generateSamples(300, 0, 2, 0.1);
}
private void printRepPoints(int numIterations) throws IOException {
for (int i = 0; i <= numIterations; i++) {
Path out = new Path(getTestTempDirPath("output"), "representativePoints-" + i);
System.out.println("Representative Points for iteration " + i);
Configuration conf = new Configuration();
for (Pair<IntWritable,VectorWritable> record :
new SequenceFileDirIterable<IntWritable,VectorWritable>(
out, PathType.LIST, PathFilters.logsCRCFilter(), null, true, conf)) {
System.out.println("\tC-" + record.getFirst().get()
+ ": " + AbstractCluster.formatVector(record.getSecond().get(), null));
}
}
}
/**
* Initialize synthetic data using 4 clusters dC units from origin having 4 representative points dP from each center
* @param dC a double cluster center offset
* @param dP a double representative point offset
* @param measure the DistanceMeasure
*/
private void initData(double dC, double dP, DistanceMeasure measure) {
clusters = new ArrayList<Cluster>();
clusters.add(new Canopy(new DenseVector(new double[] { -dC, -dC }), 1, measure));
clusters.add(new Canopy(new DenseVector(new double[] { -dC, dC }), 3, measure));
clusters.add(new Canopy(new DenseVector(new double[] { dC, dC }), 5, measure));
clusters.add(new Canopy(new DenseVector(new double[] { dC, -dC }), 7, measure));
representativePoints = new HashMap<Integer, List<VectorWritable>>();
for (Cluster cluster : clusters) {
List<VectorWritable> points = new ArrayList<VectorWritable>();
representativePoints.put(cluster.getId(), points);
points.add(new VectorWritable(cluster.getCenter().clone()));
points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { dP, dP }))));
points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { dP, -dP }))));
points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { -dP, -dP }))));
points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { -dP, dP }))));
}
}
@Test
public void testRepresentativePoints() throws Exception {
ClusteringTestUtils.writePointsToFile(referenceData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
Configuration conf = new Configuration();
// run using MR reference point calculation
CanopyDriver.run(conf, testdata, output, measure, 3.1, 1.1, true, true);
int numIterations = 2;
Path clustersIn = new Path(output, "clusters-0");
RepresentativePointsDriver.run(conf, clustersIn, new Path(output, "clusteredPoints"), output, measure, numIterations, false);
printRepPoints(numIterations);
ClusterEvaluator evaluatorMR = new ClusterEvaluator(conf, clustersIn);
// now run again using sequential reference point calculation
HadoopUtil.delete(conf, output);
CanopyDriver.run(conf, testdata, output, measure, 3.1, 1.1, true, true);
RepresentativePointsDriver.run(conf, clustersIn, new Path(output, "clusteredPoints"), output, measure, numIterations, true);
printRepPoints(numIterations);
ClusterEvaluator evaluatorSeq = new ClusterEvaluator(conf, clustersIn);
// compare results
assertEquals("InterCluster Density", evaluatorMR.interClusterDensity(), evaluatorSeq.interClusterDensity(), EPSILON);
assertEquals("IntraCluster Density", evaluatorMR.intraClusterDensity(), evaluatorSeq.intraClusterDensity(), EPSILON);
}
@Test
public void testCluster0() throws IOException {
ClusteringTestUtils.writePointsToFile(referenceData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
initData(1, 0.25, measure);
ClusterEvaluator evaluator = new ClusterEvaluator(representativePoints, clusters, measure);
assertEquals("inter cluster density", 0.33333333333333315, evaluator.interClusterDensity(), EPSILON);
assertEquals("intra cluster density", 0.3656854249492381, evaluator.intraClusterDensity(), EPSILON);
}
@Test
public void testCluster1() throws IOException {
ClusteringTestUtils.writePointsToFile(referenceData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
initData(1, 0.5, measure);
ClusterEvaluator evaluator = new ClusterEvaluator(representativePoints, clusters, measure);
assertEquals("inter cluster density", 0.33333333333333315, evaluator.interClusterDensity(), EPSILON);
assertEquals("intra cluster density", 0.3656854249492381, evaluator.intraClusterDensity(), EPSILON);
}
@Test
public void testCluster2() throws IOException {
ClusteringTestUtils.writePointsToFile(referenceData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
initData(1, 0.75, measure);
ClusterEvaluator evaluator = new ClusterEvaluator(representativePoints, clusters, measure);
assertEquals("inter cluster density", 0.33333333333333315, evaluator.interClusterDensity(), EPSILON);
assertEquals("intra cluster density", 0.3656854249492381, evaluator.intraClusterDensity(), EPSILON);
}
@Test
public void testEmptyCluster() throws IOException {
ClusteringTestUtils.writePointsToFile(referenceData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
initData(1, 0.25, measure);
Canopy cluster = new Canopy(new DenseVector(new double[] { 10, 10 }), 19, measure);
clusters.add(cluster);
List<VectorWritable> points = new ArrayList<VectorWritable>();
representativePoints.put(cluster.getId(), points);
ClusterEvaluator evaluator = new ClusterEvaluator(representativePoints, clusters, measure);
assertEquals("inter cluster density", 0.33333333333333315, evaluator.interClusterDensity(), EPSILON);
assertEquals("intra cluster density", 0.3656854249492381, evaluator.intraClusterDensity(), EPSILON);
}
@Test
public void testSingleValueCluster() throws IOException {
ClusteringTestUtils.writePointsToFile(referenceData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
initData(1, 0.25, measure);
Canopy cluster = new Canopy(new DenseVector(new double[] { 0, 0 }), 19, measure);
clusters.add(cluster);
List<VectorWritable> points = new ArrayList<VectorWritable>();
points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { 1, 1 }))));
representativePoints.put(cluster.getId(), points);
ClusterEvaluator evaluator = new ClusterEvaluator(representativePoints, clusters, measure);
assertEquals("inter cluster density", 0.33333333333333315, evaluator.interClusterDensity(), EPSILON);
assertEquals("intra cluster density", 0.3656854249492381, evaluator.intraClusterDensity(), EPSILON);
}
/**
* Representative points extraction will duplicate the cluster center if the cluster has no
* assigned points. These clusters should be ignored like empty clusters above
* @throws IOException
*/
@Test
public void testAllSameValueCluster() throws IOException {
ClusteringTestUtils.writePointsToFile(referenceData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
initData(1, 0.25, measure);
Canopy cluster = new Canopy(new DenseVector(new double[] { 0, 0 }), 19, measure);
clusters.add(cluster);
List<VectorWritable> points = new ArrayList<VectorWritable>();
points.add(new VectorWritable(cluster.getCenter()));
points.add(new VectorWritable(cluster.getCenter()));
points.add(new VectorWritable(cluster.getCenter()));
representativePoints.put(cluster.getId(), points);
ClusterEvaluator evaluator = new ClusterEvaluator(representativePoints, clusters, measure);
assertEquals("inter cluster density", 0.33333333333333315, evaluator.interClusterDensity(), EPSILON);
assertEquals("intra cluster density", 0.3656854249492381, evaluator.intraClusterDensity(), EPSILON);
}
@Test
public void testCanopy() throws Exception {
ClusteringTestUtils.writePointsToFile(sampleData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
Configuration conf = new Configuration();
CanopyDriver.run(conf, testdata, output, measure, 3.1, 1.1, true, true);
int numIterations = 10;
Path clustersIn = new Path(output, "clusters-0");
RepresentativePointsDriver.run(conf, clustersIn, new Path(output, "clusteredPoints"), output, measure, numIterations, true);
ClusterEvaluator evaluator = new ClusterEvaluator(conf, clustersIn);
// now print out the Results
System.out.println("Intra-cluster density = " + evaluator.intraClusterDensity());
System.out.println("Inter-cluster density = " + evaluator.interClusterDensity());
printRepPoints(numIterations);
}
@Test
public void testKmeans() throws Exception {
ClusteringTestUtils.writePointsToFile(sampleData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
// now run the Canopy job to prime kMeans canopies
Configuration conf = new Configuration();
CanopyDriver.run(conf, testdata, output, measure, 3.1, 1.1, false, true);
// now run the KMeans job
KMeansDriver.run(testdata, new Path(output, "clusters-0"), output, measure, 0.001, 10, true, true);
int numIterations = 10;
Path clustersIn = new Path(output, "clusters-2");
RepresentativePointsDriver.run(conf, clustersIn, new Path(output, "clusteredPoints"), output, measure, numIterations, true);
ClusterEvaluator evaluator = new ClusterEvaluator(conf, clustersIn);
// now print out the Results
System.out.println("Intra-cluster density = " + evaluator.intraClusterDensity());
System.out.println("Inter-cluster density = " + evaluator.interClusterDensity());
printRepPoints(numIterations);
}
@Test
public void testFuzzyKmeans() throws Exception {
ClusteringTestUtils.writePointsToFile(sampleData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
// now run the Canopy job to prime kMeans canopies
Configuration conf = new Configuration();
CanopyDriver.run(conf, testdata, output, measure, 3.1, 1.1, false, true);
// now run the KMeans job
FuzzyKMeansDriver.run(testdata, new Path(output, "clusters-0"), output, measure, 0.001, 10, 2, true, true, 0, true);
int numIterations = 10;
Path clustersIn = new Path(output, "clusters-4");
RepresentativePointsDriver.run(conf, clustersIn, new Path(output, "clusteredPoints"), output, measure, numIterations, true);
ClusterEvaluator evaluator = new ClusterEvaluator(conf, clustersIn);
// now print out the Results
System.out.println("Intra-cluster density = " + evaluator.intraClusterDensity());
System.out.println("Inter-cluster density = " + evaluator.interClusterDensity());
printRepPoints(numIterations);
}
@Test
public void testMeanShift() throws Exception {
ClusteringTestUtils.writePointsToFile(sampleData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
Configuration conf = new Configuration();
new MeanShiftCanopyDriver().run(conf, testdata, output, measure, 2.1, 1.0, 0.001, 10, false, true, true);
int numIterations = 10;
Path clustersIn = new Path(output, "clusters-10");
RepresentativePointsDriver.run(conf, clustersIn, new Path(output, "clusteredPoints"), output, measure, numIterations, true);
ClusterEvaluator evaluator = new ClusterEvaluator(conf, clustersIn);
// now print out the Results
System.out.println("Intra-cluster density = " + evaluator.intraClusterDensity());
System.out.println("Inter-cluster density = " + evaluator.interClusterDensity());
printRepPoints(numIterations);
}
@Test
public void testDirichlet() throws Exception {
ClusteringTestUtils.writePointsToFile(sampleData, new Path(testdata, "file1"), fs, conf);
DistributionDescription description =
new DistributionDescription(GaussianClusterDistribution.class.getName(),
DenseVector.class.getName(),
null,
2);
DirichletDriver.run(testdata, output, description, 15, 5, 1.0, true, true, 0, true);
int numIterations = 10;
Configuration conf = new Configuration();
Path clustersIn = new Path(output, "clusters-5");
RepresentativePointsDriver.run(conf,
clustersIn,
new Path(output, "clusteredPoints"),
output,
new EuclideanDistanceMeasure(),
numIterations,
true);
ClusterEvaluator evaluator = new ClusterEvaluator(conf, clustersIn);
// now print out the Results
System.out.println("Intra-cluster density = " + evaluator.intraClusterDensity());
System.out.println("Inter-cluster density = " + evaluator.interClusterDensity());
printRepPoints(numIterations);
}
}