LOG.info("Construct a shard writer for " + key);
FileSystem fs = FileSystem.get(iconf.getConfiguration());
String temp =
mapredTempDir + Path.SEPARATOR + "shard_" + System.currentTimeMillis();
final ShardWriter writer = new ShardWriter(fs, key, temp, iconf);
// update the shard
while (values.hasNext()) {
IntermediateForm form = values.next();
writer.process(form);
reporter.progress();
}
// close the shard
final Reporter fReporter = reporter;
new Closeable() {
volatile boolean closed = false;
public void close() throws IOException {
// spawn a thread to give progress heartbeats
Thread prog = new Thread() {
public void run() {
while (!closed) {
try {
fReporter.setStatus("closing");
Thread.sleep(1000);
} catch (InterruptedException e) {
continue;
} catch (Throwable e) {
return;
}
}
}
};
try {
prog.start();
if (writer != null) {
writer.close();
}
} finally {
closed = true;
}
}