while (!tablesNotProcessed.isEmpty()) {
String table = tablesNotProcessed.peek();
String selectQuery = String.format(MigrationQuery.SELECT_RAW_DATA.toString(), table);
ExistingDataSource dataSource = getExistingDataSource(selectQuery, task, config);
dataSource.initialize();
log.info("Start migrating raw table: " + table);
telemetry.getMigrationTimer().resume();
int lastMigratedRecord = 0;
while (true) {
existingData = dataSource.getData(lastMigratedRecord, MAX_RECORDS_TO_LOAD_FROM_SQL);
if (existingData == null || existingData.size() == 0) {
break;
}
lastMigratedRecord += existingData.size();
failureCount = 0;
while (failureCount < MAX_NUMBER_OF_FAILURES) {
try {
insertDataToCassandra(existingData);
break;
} catch (Exception e) {
log.error("Failed to insert " + MetricsTable.RAW.toString()
+ " data. Attempting to insert the current batch of data one more time");
log.error(e);
failureCount++;
if (failureCount == MAX_AGGREGATE_BATCH_TO_CASSANDRA) {
throw e;
}
}
}
log.info("- " + table + " - " + lastMigratedRecord + " -");
numberOfBatchesMigrated++;
if (Task.Estimate.equals(task) && numberOfBatchesMigrated >= NUMBER_OF_BATCHES_FOR_ESTIMATION) {
break;
}
}
telemetry.getMigrationTimer().suspend();
if (Task.Migrate.equals(task)) {
log.info("Done migrating raw table" + table + "---------------------");
if (config.isDeleteDataImmediatelyAfterMigration()) {
deleteTableData(table);
}
} else if (numberOfBatchesMigrated >= NUMBER_OF_BATCHES_FOR_ESTIMATION) {
break;
}
dataSource.close();
tablesNotProcessed.poll();
}
telemetry.getMigrationTimer().resume();
metricsIndexAccumulator.drain();