///////////////////////////////////////////////////////////////////////
// Note: Remove any communityids already in the source and set groupdID to
// the communityid param (supports multiple IDs in a comma separated list)
communityIdSet.add(new ObjectId(communityIdStr));
source = ApiManager.mapFromApi(sourceString, SourcePojo.class, new SourcePojoApiMap(communityIdSet));
if (null == source.getCommunityIds()) {
source.setCommunityIds(new HashSet<ObjectId>());
}
for (ObjectId sid: communityIdSet) {
source.getCommunityIds().add(sid);
}
source.setFederatedQueryCommunityIds(null); // (can be filled in by fillInSourcePipelineFields() below)
source.fillInSourcePipelineFields(); // (needs to be after the community ids)
// RSS search harvest types tend to be computationally expensive and therefore
// should be done less frequently (by default once/4-hours seems good):
if (sourceSearchesWeb(source)) {
// If the search cycle has not been specified, use a default:
if (null == source.getSearchCycle_secs()) {
source.setSearchCycle_secs(4*3600); // (ie 4 hours)
}
}
//TESTED
}
catch (Exception e)
{
rp.setResponse(new ResponseObject("Source", false, "Unable to serialize Source JSON. Error: " + e.getMessage()));
return rp;
}
BasicDBObject query = null;
//SPECIAL CASE: IF AN ACTIVE LOGSTASH HARVEST THEN CHECK BEFORE WE'LL PUBLISH:
if (source.getExtractType().equalsIgnoreCase("logstash") &&
((null == source.getSearchCycle_secs()) || (source.getSearchCycle_secs() > 0)))
{
ResponsePojo rpTest = this.testSource(sourceString, 0, false, false, userIdStr);
if (!rpTest.getResponse().isSuccess()) {
rp.setResponse(new ResponseObject("Source", false, "Logstash not publishable. Error: " + rpTest.getResponse().getMessage()));
return rp;
}
}//TESTED
///////////////////////////////////////////////////////////////////////
// If source._id == null this should be a new source
if ((source.getId() == null) && (source.getKey() == null))
{
///////////////////////////////////////////////////////////////////////
// Note: Overwrite the following fields regardless of what was sent in
source.setId(new ObjectId());
source.setOwnerId(new ObjectId(userIdStr));
source.setApproved(isApproved);
source.setCreated(new Date());
source.setModified(new Date());
source.setUrl(source.getUrl());
// (key generated below from representative URL - don't worry about the fact this field is sometimes not present)
source.setKey(validateSourceKey(source.getId(), source.generateSourceKey()));
source.generateShah256Hash();
// Note: Create/update the source's Shah-256 hash
///////////////////////////////////////////////////////////////////////
// Note: Check the SourcePojo to make sure the required fields are there
// return an error message to the user if any are missing
String missingFields = hasRequiredSourceFields(source);
if (missingFields != null && missingFields.length() > 0)
{
rp.setResponse(new ResponseObject("Source", false, missingFields));
return rp;
}
///////////////////////////////////////////////////////////////////////
// Add the new source to the harvester.sources collection
try
{
// Need to double check that the community has an index (for legacy reasons):
if (null == DbManager.getIngest().getSource().findOne(new BasicDBObject(SourcePojo.communityIds_,
new BasicDBObject(MongoDbManager.in_, source.getCommunityIds()))))
{
for (ObjectId id: source.getCommunityIds()) {
GenericProcessingController.recreateCommunityDocIndex_unknownFields(id, false);
}
}
//TESTED
DbManager.getIngest().getSource().save(source.toDb());
if (isUniqueSource(source, communityIdSet))
{
rp.setResponse(new ResponseObject("Source", true, "New source added successfully."));
}
else { // Still allow people to add identical sources, but warn them so they can delete it if they way
rp.setResponse(new ResponseObject("Source", true, "New source added successfully. Note functionally identical sources are also present within your communities, which may waste system resources."));
}
rp.setData(source, new SourcePojoApiMap(null, communityIdSet, null));
}
catch (Exception e)
{
rp.setResponse(new ResponseObject("Source", false, "Unable to add new source. Error: " + e.getMessage()));
}
///////////////////////////////////////////////////////////////////////
// If the user is not the owner or moderator we need to send the owner
// and email asking them to approve or reject the source
try {
if (!isApproved)
{
emailSourceApprovalRequest(source);
}
}
catch (Exception e) { // Unable to ask for permission, remove sources and error out
logger.error("Exception Message: " + e.getMessage(), e);
DbManager.getIngest().getSource().remove(new BasicDBObject(SourcePojo._id_, source.getId()));
rp.setData((String)null, (BasePojoApiMap<String>)null); // (unset)
rp.setResponse(new ResponseObject("Source", false, "Unable to email authority for permission, maybe email infrastructure isn't added? Error: " + e.getMessage()));
}
}//TESTED (behavior when an identical source is added)
///////////////////////////////////////////////////////////////////////
// Existing source, update if possible
else
{
if ((null != source.getPartiallyPublished()) && source.getPartiallyPublished()) {
rp.setResponse(new ResponseObject("Source", false, "Unable to update source - the source is currently in 'partially published' mode, because it is private and you originally accessed it without sufficient credentials. Make a note of your changes, revert, and try again."));
return rp;
}//TESTED
///////////////////////////////////////////////////////////////////////
// Attempt to retrieve existing share from harvester.sources collection
query = new BasicDBObject();
if (null != source.getId()) {
query.put(SourcePojo._id_, source.getId());
}
else if (null != source.getKey()) {
query.put(SourcePojo.key_, source.getKey());
}
try
{
BasicDBObject dbo = (BasicDBObject)DbManager.getIngest().getSource().findOne(query);
// Source doesn't exist so it can't be updated
if (dbo == null)
{
rp.setResponse(new ResponseObject("Source", false, "Unable to update source. The source ID is invalid."));
return rp;
}
SourcePojo oldSource = SourcePojo.fromDb(dbo,SourcePojo.class);
///////////////////////////////////////////////////////////////////////
// Note: Only an Infinit.e administrator, source owner, community owner
// or moderator can update/edit a source
if (null == oldSource.getOwnerId()) { // (internal error, just correct)
oldSource.setOwnerId(new ObjectId(userIdStr));
}
boolean isSourceOwner = oldSource.getOwnerId().toString().equalsIgnoreCase(userIdStr);
if (!isSourceOwner) {
boolean ownerModOrApprovedSysAdmin = isApproved &&
(SocialUtils.isOwnerOrModerator(communityIdStr, userIdStr) || RESTTools.adminLookup(userIdStr));
if (!ownerModOrApprovedSysAdmin)
{
rp.setResponse(new ResponseObject("Source", false, "User does not have permissions to edit this source"));
return rp;
}
}//TESTED - works if owner or moderator, or admin (enabled), not if not admin-enabled
// For now, don't allow you to change communities
if ((null == source.getCommunityIds()) || (null == oldSource.getCommunityIds()) // (robustness)
||
!source.getCommunityIds().equals(oldSource.getCommunityIds()))
{
rp.setResponse(new ResponseObject("Source", false, "It is not currently possible to change the community of a published source. You must duplicate/scrub the source and re-publish it as a new source (and potentially suspend/delete this one)"));
return rp;
}//TOTEST
//isOwnerOrModerator
String oldHash = source.getShah256Hash();
///////////////////////////////////////////////////////////////////////
// Note: The following fields in an existing source cannot be changed: Key
// Make sure new source url and key match existing source values
// (we allow URL to be changed, though obv the key won't be changed to reflect that)
source.setKey(oldSource.getKey());
// Overwrite/set other values in the new source from old source as appropriate
source.setCreated(oldSource.getCreated());
source.setModified(new Date());
source.setOwnerId(oldSource.getOwnerId());
if (null == source.getIsPublic()) {
source.setIsPublic(oldSource.getIsPublic());
}//TESTED
// Harvest status specification logic (we need normally need to keep these fields intact):
// - If harvest completely unspecified, delete everything but num records
// - If harvest specified, and there exists an existing harvest block then ignore
// - If harvest specified, and the harvest has previously been deleted, then copy (except num records)
// - Extra ... if new status object has harvested unset, then unset that
if ((null == source.getHarvestStatus()) && (null != oldSource.getHarvestStatus())) {
// request to unset the harvest status altogether
source.setHarvestStatus(new SourceHarvestStatusPojo()); // new harvest status
source.getHarvestStatus().setDoccount(oldSource.getHarvestStatus().getDoccount());
// but keep number of records
}
else if ((null != oldSource.getHarvestStatus()) && (null == oldSource.getHarvestStatus().getHarvest_status())) {
// Has previously been unset with the logic from the above clause
source.getHarvestStatus().setDoccount(oldSource.getHarvestStatus().getDoccount());
// (null != source.getHarvestStatus()) else would be in the clause above
}
else if (null != oldSource.getHarvestStatus()) {
// Unset the harvested time to queue a new harvest cycle
if ((null != source.getHarvestStatus()) && (null == source.getHarvestStatus().getHarvested())) {
oldSource.getHarvestStatus().setHarvested(null);
}
source.setHarvestStatus(oldSource.getHarvestStatus());
}
//(else oldSource.getHarvestStatus is null, just retain the updated version)
//TESTED: no original harvest status, failing to edit existing harvest status, delete status (except doc count), update deleted status (except doc count)
// If we're changing the distribution factor, need to keep things a little bit consistent:
if ((null == source.getDistributionFactor()) && (null != oldSource.getDistributionFactor())) {
// Removing it:
if (null != source.getHarvestStatus()) {
source.getHarvestStatus().setDistributionReachedLimit(null);
source.getHarvestStatus().setDistributionTokensComplete(null);
source.getHarvestStatus().setDistributionTokensFree(null);
}
}//TESTED
else if ((null != source.getDistributionFactor()) && (null != oldSource.getDistributionFactor())
&& (source.getDistributionFactor() != oldSource.getDistributionFactor()))
{
// Update the number of available tokens:
if ((null != source.getHarvestStatus()) && (null != source.getHarvestStatus().getDistributionTokensFree()))
{
int n = source.getHarvestStatus().getDistributionTokensFree() +
(source.getDistributionFactor() - oldSource.getDistributionFactor());
if (n < 0) n = 0;
source.getHarvestStatus().setDistributionTokensFree(n);
}
}//TESTED
///////////////////////////////////////////////////////////////////////
// Check for missing fields:
String missingFields = hasRequiredSourceFields(source);
if (missingFields != null && missingFields.length() > 0)
{
rp.setResponse(new ResponseObject("Source", false, missingFields));
return rp;
}
///////////////////////////////////////////////////////////////////////
// Note: Create/update the source's Shah-256 hash
source.generateShah256Hash();
///////////////////////////////////////////////////////////////////////
// Handle approval:
if (isApproved || oldHash.equalsIgnoreCase(source.getShah256Hash())) {
//(either i have permissions, or the source hasn't change)
if (oldSource.isApproved()) { // Always approve - annoyingly no way of unsetting this
source.setApproved(true);
}
else if (source.isApproved()) { // Want to re-approve
if (!isApproved) // Don't have permission, so reset
{
source.setApproved(oldSource.isApproved());
}
}
}
else { // Need to re-approve
try {
source.setApproved(false);
emailSourceApprovalRequest(source);
}
catch (Exception e) { // Unable to ask for permission, remove sources and error out
logger.error("Exception Message: " + e.getMessage(), e);
DbManager.getIngest().getSource().remove(new BasicDBObject(SourcePojo._id_, source.getId()));
rp.setData((String)null, (BasePojoApiMap<String>)null); // (unset)
rp.setResponse(new ResponseObject("Source", false, "Unable to email authority for permission, maybe email infrastructure isn't added? Error: " + e.getMessage()));
}
}//TOTEST
// Source exists, update and prepare reply
DbManager.getIngest().getSource().update(query, source.toDb());
if (isUniqueSource(source, communityIdSet))
{
rp.setResponse(new ResponseObject("Source", true, "Source has been updated successfully."));
}
else { // Still allow people to add identical sources, but warn them so they can delete it if they way
rp.setResponse(new ResponseObject("Source", true, "Source has been updated successfully. Note functionally identical sources are also present within your communities, which may waste system resources."));
}
rp.setData(source, new SourcePojoApiMap(null, communityIdSet, null));
}
catch (Exception e)
{
logger.error("Exception Message: " + e.getMessage(), e);
rp.setResponse(new ResponseObject("Source", false, "Unable to update source: " + e.getMessage()));