package edu.isi.karma.controller.command.alignment;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import java.util.Stack;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import edu.isi.karma.controller.command.Command;
import edu.isi.karma.controller.command.CommandException;
import edu.isi.karma.controller.command.CommandType;
import edu.isi.karma.controller.command.WorksheetSelectionCommand;
import edu.isi.karma.controller.command.selection.SuperSelection;
import edu.isi.karma.controller.command.worksheet.AddValuesCommand;
import edu.isi.karma.controller.command.worksheet.AddValuesCommandFactory;
import edu.isi.karma.controller.update.ErrorUpdate;
import edu.isi.karma.controller.update.UpdateContainer;
import edu.isi.karma.controller.update.WorksheetUpdateFactory;
import edu.isi.karma.er.helper.CloneTableUtils;
import edu.isi.karma.er.helper.TripleStoreUtil;
import edu.isi.karma.kr2rml.ErrorReport;
import edu.isi.karma.kr2rml.URIFormatter;
import edu.isi.karma.modeling.Uris;
import edu.isi.karma.modeling.alignment.Alignment;
import edu.isi.karma.modeling.alignment.AlignmentManager;
import edu.isi.karma.modeling.ontology.OntologyManager;
import edu.isi.karma.rep.HNode;
import edu.isi.karma.rep.HNode.HNodeType;
import edu.isi.karma.rep.HashValueManager;
import edu.isi.karma.rep.Node;
import edu.isi.karma.rep.RepFactory;
import edu.isi.karma.rep.Row;
import edu.isi.karma.rep.Table;
import edu.isi.karma.rep.Worksheet;
import edu.isi.karma.rep.Workspace;
import edu.isi.karma.rep.alignment.Label;
import edu.isi.karma.rep.alignment.SemanticType.ClientJsonKeys;
import edu.isi.karma.rep.metadata.WorksheetProperties.Property;
import edu.isi.karma.webserver.KarmaException;
public class AugmentDataCommand extends WorksheetSelectionCommand{
private static Logger LOG = LoggerFactory.getLogger(AugmentDataCommand.class);
private String predicate;
private String columnUri;
private String alignmentId;
private String otherClass;
private String dataRepoUrl;
private String hNodeId;
private String newhNodeId;
private boolean incoming;
private String sameAsPredicate;
private final Integer limit = 200;
Stack<Command> appliedCommands;
public AugmentDataCommand(String id, String dataRepoUrl, String worksheetId, String columnUri, String predicate, String otherClass, String hNodeId, Boolean incoming, String sameAsPredicate, String selectionId) {
super(id, worksheetId, selectionId);
this.predicate = predicate;
this.columnUri = columnUri;
this.otherClass = otherClass;
this.dataRepoUrl = dataRepoUrl;
this.hNodeId = hNodeId;
newhNodeId = hNodeId;
this.incoming = incoming;
this.sameAsPredicate = sameAsPredicate;
appliedCommands = new Stack<Command>();
addTag(CommandTag.Transformation);
}
@Override
public String getCommandName() {
return this.getClass().getSimpleName();
}
@Override
public String getTitle() {
return "Augment Data";
}
@Override
public String getDescription() {
JSONArray predicatesarray = new JSONArray(predicate);
StringBuilder builder = new StringBuilder();
for(int i = 0; i < predicatesarray.length() - 1; i++)
builder.append(predicatesarray.getJSONObject(i).getString("predicate")).append(" ");
builder.append(predicatesarray.getJSONObject(predicatesarray.length() - 1).getString("predicate"));
return builder.toString();
}
@Override
public CommandType getCommandType() {
return CommandType.undoable;
}
@Override
public UpdateContainer doIt(Workspace workspace) throws CommandException {
appliedCommands.clear();
inputColumns.clear();
outputColumns.clear();
UpdateContainer c = new UpdateContainer();
alignmentId = AlignmentManager.Instance().constructAlignmentId(workspace.getId(), worksheetId);
Alignment alignment = AlignmentManager.Instance().getAlignment(alignmentId);
RepFactory factory = workspace.getFactory();
Worksheet worksheet = factory.getWorksheet(worksheetId);
SuperSelection selection = getSuperSelection(worksheet);
if (alignment.GetTreeRoot() != null)
hNodeId = FetchHNodeIdFromAlignmentCommand.gethNodeId(alignmentId, columnUri);
if (hNodeId == null) {
c.append(WorksheetUpdateFactory.createRegenerateWorksheetUpdates(worksheetId, getSuperSelection(worksheet)));
c.append(computeAlignmentAndSemanticTypesAndCreateUpdates(workspace));
return c;
}
HNode hnode = factory.getHNode(hNodeId);
List<String> hNodeIds = new LinkedList<String>();
hNodeIds.add(hNodeId);
inputColumns.addAll(hNodeIds);
List<Table> dataTables = new ArrayList<Table>();
CloneTableUtils.getDatatable(worksheet.getDataTable(), factory.getHTable(hnode.getHTableId()), dataTables, selection);
Map<String, String> rowHashToSubjectURI = new HashMap<String, String>();
Map<String, List<String>> SubjectURIToRowId = new HashMap<String, List<String>>();
for(Table t : dataTables) {
for(Row r : t.getRows(0, t.getNumRows(), selection)) {
Node n = r.getNode(hNodeId);
if(n != null && n.getValue() != null && !n.getValue().isEmptyValue() && n.getValue().asString() != null && !n.getValue().asString().trim().isEmpty() ) {
String uri = n.getValue().asString().trim().replace(" ", "");
String baseURI = worksheet.getMetadataContainer().getWorksheetProperties().getPropertyValue(Property.baseURI);
try {
URI t1 = new URI(uri);
if (!t1.isAbsolute() && baseURI != null) {
uri = baseURI + uri;
}
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
}
// n.setValue(uri, n.getStatus(), factory);
rowHashToSubjectURI.put(HashValueManager.getHashValue(r, hNodeIds), uri);
if (SubjectURIToRowId.get(uri) == null)
SubjectURIToRowId.put(uri, new ArrayList<String>());
List<String> rowIds = SubjectURIToRowId.get(uri);
rowIds.add(r.getId());
}
}
}
TripleStoreUtil util = new TripleStoreUtil();
//String modelContext = worksheet.getMetadataContainer().getWorksheetProperties().getPropertyValue(Property.modelContext);
List<String> subjects = new LinkedList<String>();
subjects.addAll(rowHashToSubjectURI.values());
List<String> predicates = new LinkedList<String>();
List<String> otherClasses = new LinkedList<String>();
Map<String, List<String>> results = new HashMap<String, List<String>>();
URIFormatter uriFormatter = new URIFormatter(workspace.getOntologyManager(), new ErrorReport());
if(sameAsPredicate!= null && !sameAsPredicate.trim().isEmpty())
{
sameAsPredicate = uriFormatter.getExpandedAndNormalizedUri(sameAsPredicate);
}
JSONArray predicatesarray = new JSONArray(predicate);
JSONArray otherClassarray = new JSONArray(otherClass);
for(int i = 0; i < predicatesarray.length(); i++) {
predicates.add(predicatesarray.getJSONObject(i).getString("predicate"));
otherClasses.add(otherClassarray.getJSONObject(i).getString("otherClass"));
}
while (subjects.size() > 0) {
ListIterator<String> subjectsIterator = subjects.listIterator();
LinkedList<String> tempSubjects = new LinkedList<String>();
while(tempSubjects.size() < limit && subjects.size() > 0)
{
tempSubjects.add(subjectsIterator.next());
subjectsIterator.remove();
}
try {
Map<String, List<String>> temp = null;
if (!incoming)
temp = util.getObjectsForSubjectsAndPredicates(dataRepoUrl, null, tempSubjects , predicates, otherClasses, sameAsPredicate);
else
temp = util.getSubjectsForPredicatesAndObjects(dataRepoUrl, null, tempSubjects , predicates, otherClasses, sameAsPredicate);
addMappingToResults(results, temp);
// predicates.clear();
// otherClasses.clear();
} catch (KarmaException e) {
LOG.error("Unable to load data to augment: ", e);
return new UpdateContainer(new ErrorUpdate(e.getMessage()));
}
}
List<String> resultSubjects = results.get("resultSubjects");
List<String> resultPredicates = results.get("resultPredicates");
List<String> resultObjects = results.get("resultObjects");
List<String> resultClass = results.get("resultClasses");
AddValuesCommandFactory addFactory = new AddValuesCommandFactory();
for (int i = 0; i < resultPredicates.size(); i++) {
String subject = incoming ? resultObjects.get(i) : resultSubjects.get(i);
List<String> rowIds = SubjectURIToRowId.get(subject);
boolean isNewNode = false;
for (String RowId : rowIds) {
String predicate = resultPredicates.get(i);
String otherClass = resultClass.get(i);
JSONArray array = new JSONArray();
JSONObject obj = new JSONObject();
JSONObject obj2 = new JSONObject();
if (otherClass != null && !otherClass.trim().isEmpty())
obj.put("URIs", incoming ? resultSubjects.get(i) : resultObjects.get(i));
else
obj.put("values", incoming ? resultSubjects.get(i) : resultObjects.get(i));
obj2.put("rowId", RowId);
obj2.put("rowIdHash", "");
obj2.put("values", obj);
array.put(obj2);
JSONArray input = new JSONArray();
JSONObject obj3 = new JSONObject();
obj3.put("name", "AddValues");
obj3.put("value", array.toString());
obj3.put("type", "other");
input.put(obj3);
try {
OntologyManager ontMgr = workspace.getOntologyManager();
Label label = ontMgr.getUriLabel(incoming ? otherClass : predicate);
AddValuesCommand command = (AddValuesCommand) addFactory.createCommand(input, workspace, hNodeId, worksheetId, hnode.getHTableId(), label.getDisplayName(), HNodeType.AugmentData, selection.getName());
command.doIt(workspace);
outputColumns.addAll(command.getOutputColumns());
isNewNode |= command.isNewNode();
if (command.isNewNode())
appliedCommands.push(command);
newhNodeId = command.getNewHNodeId();
} catch(Exception e) {
e.printStackTrace();
return new UpdateContainer(new ErrorUpdate(e.getMessage()));
}
}
if (isNewNode && alignment.GetTreeRoot() != null) {
HNode tableHNode =workspace.getFactory().getHNode(newhNodeId);
String nestedHNodeId = tableHNode.getNestedTable().getHNodeIdFromColumnName("values");
if (nestedHNodeId == null)
nestedHNodeId = tableHNode.getNestedTable().getHNodeIdFromColumnName("URIs");
SetSemanticTypeCommandFactory sstFactory = new SetSemanticTypeCommandFactory();
JSONArray semanticTypesArray = new JSONArray();
JSONObject semanticType = new JSONObject();
edu.isi.karma.rep.alignment.Node n = alignment.getNodeById(columnUri);
semanticType.put(ClientJsonKeys.isPrimary.name(), "true");
Set<edu.isi.karma.rep.alignment.Node> oldNodes = new HashSet<edu.isi.karma.rep.alignment.Node>();
if(resultClass.get(i).trim().isEmpty())
{
semanticType.put(ClientJsonKeys.DomainId.name(), n.getId());
semanticType.put(ClientJsonKeys.FullType.name(), resultPredicates.get(i));
semanticType.put(ClientJsonKeys.DomainUri.name(), n.getUri());
}
else
{
if (alignment.getNodesByUri(resultClass.get(i)) != null)
oldNodes.addAll( alignment.getNodesByUri(resultClass.get(i)));
semanticType.put(ClientJsonKeys.DomainId.name(), resultClass.get(i));
semanticType.put(ClientJsonKeys.FullType.name(), Uris.CLASS_INSTANCE_LINK_URI);
}
semanticTypesArray.put(semanticType);
Command sstCommand = sstFactory.createCommand(workspace, worksheetId, nestedHNodeId, false, semanticTypesArray, false, "", selection.getName());
appliedCommands.push(sstCommand);
sstCommand.doIt(workspace);
if(!resultClass.get(i).trim().isEmpty())
{
ChangeInternalNodeLinksCommandFactory cinlcf = new ChangeInternalNodeLinksCommandFactory();
SetMetaPropertyCommandFactory smpcf = new SetMetaPropertyCommandFactory();
JSONArray newEdges = new JSONArray();
JSONObject newEdge = new JSONObject();
String sourceId = n.getId();
Set<edu.isi.karma.rep.alignment.Node> tempnodes = new HashSet<edu.isi.karma.rep.alignment.Node>();
tempnodes.addAll(alignment.getNodesByUri(resultClass.get(i)));
tempnodes.removeAll(oldNodes);
edu.isi.karma.rep.alignment.Node target = tempnodes.iterator().next();
String targetId = target.getId();
String targetUri = target.getLabel().getUri();
String edgeUri = resultPredicates.get(i);
if (!incoming) {
newEdge.put(ChangeInternalNodeLinksCommand.JsonKeys.edgeSourceId.name(), sourceId);
newEdge.put(ChangeInternalNodeLinksCommand.JsonKeys.edgeTargetId.name(), targetId);
newEdge.put(ChangeInternalNodeLinksCommand.JsonKeys.edgeId.name(), edgeUri);
}
else {
newEdge.put(ChangeInternalNodeLinksCommand.JsonKeys.edgeSourceId.name(), targetId);
newEdge.put(ChangeInternalNodeLinksCommand.JsonKeys.edgeTargetId.name(), sourceId);
newEdge.put(ChangeInternalNodeLinksCommand.JsonKeys.edgeId.name(), edgeUri);
}
newEdges.put(newEdge);
Command changeInternalNodeLinksCommand = cinlcf.createCommand(worksheetId, alignmentId, new JSONArray(), newEdges, workspace);
changeInternalNodeLinksCommand.doIt(workspace);
appliedCommands.push(changeInternalNodeLinksCommand);
Command setMetaDataCommand = smpcf.createCommand(workspace, nestedHNodeId, worksheetId, "isUriOfClass",
targetUri, targetId, "", selection.getName());
setMetaDataCommand.doIt(workspace);
appliedCommands.push(setMetaDataCommand);
}
}
}
WorksheetUpdateFactory.detectSelectionStatusChange(worksheetId, workspace, this);
c.append(WorksheetUpdateFactory.createRegenerateWorksheetUpdates(worksheetId, getSuperSelection(worksheet)));
c.append(computeAlignmentAndSemanticTypesAndCreateUpdates(workspace));
return c;
}
@Override
public UpdateContainer undoIt(Workspace workspace) {
UpdateContainer c = new UpdateContainer();
while (!appliedCommands.isEmpty()) {
Command command = appliedCommands.pop();
command.undoIt(workspace);
}
c.append(WorksheetUpdateFactory.createRegenerateWorksheetUpdates(worksheetId, getSuperSelection(workspace)));
c.append(computeAlignmentAndSemanticTypesAndCreateUpdates(workspace));
return c;
}
private void addMappingToResults(Map<String, List<String>> results, Map<String, List<String>> temp) {
List<String> resultSubjects = results.get("resultSubjects");
List<String> resultPredicates = results.get("resultPredicates");
List<String> resultObjects = results.get("resultObjects");
List<String> resultClasses = results.get("resultClasses");
if (resultSubjects == null)
resultSubjects = new LinkedList<String>();
if (resultPredicates == null)
resultPredicates = new LinkedList<String>();
if (resultObjects == null)
resultObjects = new LinkedList<String>();
if (resultClasses == null)
resultClasses = new LinkedList<String>();
resultSubjects.addAll(temp.get("resultSubjects"));
resultPredicates.addAll(temp.get("resultPredicates"));
resultObjects.addAll(temp.get("resultObjects"));
resultClasses.addAll(temp.get("resultClasses"));
results.put("resultSubjects", resultSubjects);
results.put("resultPredicates", resultPredicates);
results.put("resultObjects", resultObjects);
results.put("resultClasses", resultClasses);
}
}