DistributedCache.addCacheFile(secondCacheFile.toUri(), conf1);
TrackerDistributedCacheManager.determineTimestampsAndCacheVisibilities(conf1);
// Task localizing for first job
JobID jobId = new JobID("jt", 1);
TaskDistributedCacheManager handle = manager
.newTaskDistributedCacheManager(jobId, conf1);
handle.setupCache(conf1, TaskTracker.getPublicDistributedCacheDir(),
TaskTracker.getPrivateDistributedCacheDir(userName));
long[] sizes = JobLocalizer.downloadPrivateCache(conf1);
if (sizes != null) {
manager.setArchiveSizes(jobId, sizes);
}
handle.release();
for (TaskDistributedCacheManager.CacheFile c : handle.getCacheFiles()) {
assertEquals(0, manager.getReferenceCount(c.getStatus()));
long filesize = FileUtil.getDU(new File(c.getStatus().localizedLoadPath.getParent().toString()));
assertTrue("filesize is not greater than 0", filesize > 0);
assertEquals(filesize, c.getStatus().size);
}
// Test specifying directories to go into distributed cache and make
// their sizes are calculated properly.
Job job2 = new Job(conf);
Configuration conf2 = job2.getConfiguration();
conf1.set("user.name", userName);
DistributedCache.addCacheFile(firstCacheDirPublic.toUri(), conf2);
DistributedCache.addCacheFile(firstCacheDirPrivate.toUri(), conf2);
TrackerDistributedCacheManager.determineTimestampsAndCacheVisibilities(conf2);
// Task localizing for second job
JobID job2Id = new JobID("jt", 2);
handle = manager.newTaskDistributedCacheManager(job2Id, conf2);
handle.setupCache(conf2, TaskTracker.getPublicDistributedCacheDir(),
TaskTracker.getPrivateDistributedCacheDir(userName));
long[] sizes2 = JobLocalizer.downloadPrivateCache(conf2);
for (int j=0; j > sizes2.length; j++) {
LOG.info("size is: " + sizes2[j]);
}
if (sizes2 != null) {
manager.setArchiveSizes(job2Id, sizes2);
}
handle.release();
for (TaskDistributedCacheManager.CacheFile c : handle.getCacheFiles()) {
assertEquals(0, manager.getReferenceCount(c.getStatus()));
long filesize = FileUtil.getDU(new File(c.getStatus().localizedLoadPath.getParent().toString()));
assertTrue("filesize is not greater than 0", filesize > 0);
assertEquals(filesize, c.getStatus().size);
}
Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile");
createPrivateTempFile(thirdCacheFile);
// Configures another job with three regular files.
Job job3 = new Job(conf);
Configuration conf3 = job3.getConfiguration();
conf3.set("user.name", userName);
// add a file that would get failed to localize
DistributedCache.addCacheFile(firstCacheFilePublic.toUri(), conf3);
// add a file that is already localized by different job
DistributedCache.addCacheFile(secondCacheFile.toUri(), conf3);
// add a file that is never localized
DistributedCache.addCacheFile(thirdCacheFile.toUri(), conf3);
TrackerDistributedCacheManager.determineTimestampsAndCacheVisibilities(conf3);
// Task localizing for third job
// localization for the "firstCacheFile" will fail.
handle = manager.newTaskDistributedCacheManager(new JobID("jt", 3), conf3);
Throwable th = null;
try {
handle.setupCache(conf3, TaskTracker.getPublicDistributedCacheDir(),
TaskTracker.getPrivateDistributedCacheDir(userName));
JobLocalizer.downloadPrivateCache(conf3);