System.out.println("Publishing " + csvOrTsvFile.getName() + " via delta-importer-2 over HTTP");
DatasyncDirectory datasyncDir = new DatasyncDirectory(http, domain, datasetId);
boolean useCompression = true;
InputStream previousSignature = null;
SizeCountingInputStream patch = null;
int chunkSize = fetchDatasyncChunkSize();
String uuid = controlFile.generateAndAddOpaqueUUID();
int retryCount = 0;
do {
try {
// get signature of previous csv/tsv file
pathToSignature = datasyncDir.getPathToSignature();
previousSignature = getPreviousSignature(pathToSignature);
final long fileSize = csvOrTsvFile.length();
InputStream progressingInputStream = new ProgressingInputStream(new FileInputStream(csvOrTsvFile)) {
@Override
protected void progress(long count) {
System.out.println("\tRead " + count + " of " + fileSize + " bytes of " + csvOrTsvFile.getName());
}
};
// compute the patch between the csv/tsv file and its previous signature
patch = new SizeCountingInputStream(getPatch(progressingInputStream, previousSignature, chunkSize, useCompression));
// post the patch file in blobby chunks - ewww
List<String> blobIds = postPatchBlobs(patch, datasetId, chunkSize);
// commit the chunks, thereby applying the diff
CommitMessage commit = new CommitMessage()
.filename(csvOrTsvFile.getName() + patchExtenstion + (useCompression ? compressionExtenstion : ""))
.relativeTo(pathToSignature)
.chunks(blobIds)
.control(controlFile)
.expectedSize(patch.getTotal());
String jobId = commitBlobPostings(commit, datasetId, uuid);
// return status
return getJobStatus(datasetId, jobId);
} catch (CompletelyRestartJob e) {
retryCount += 1;
} catch (ParseException | NoSuchAlgorithmException | InputException | URISyntaxException |
SignatureException |InterruptedException | HttpException e) {
e.printStackTrace();
JobStatus jobStatus = JobStatus.PUBLISH_ERROR;
jobStatus.setMessage(e.getMessage());
return jobStatus;
} finally {
if (previousSignature != null) { previousSignature.close(); }
if (patch != null) { patch.close(); }
if (signatureResponse != null) { signatureResponse.close(); }
}
} while(retryCount < httpRetries);
JobStatus jobStatus = JobStatus.PUBLISH_ERROR;
jobStatus.setMessage("Couldn't get the request through; too many retries"); // TODO Better message