/* $Id: ManifoldCF.java 996524 2010-09-13 13:38:01Z kwright $ */
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.manifoldcf.crawler.system;
import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.crawler.interfaces.*;
import org.apache.manifoldcf.authorities.interfaces.*;
import java.io.*;
import java.security.GeneralSecurityException;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.InvalidParameterException;
import java.security.Key;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.KeySpec;
import java.util.*;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.SecretKeySpec;
public class ManifoldCF extends org.apache.manifoldcf.agents.system.ManifoldCF
{
public static final String _rcsid = "@(#)$Id: ManifoldCF.java 996524 2010-09-13 13:38:01Z kwright $";
// Initialization flag.
protected static boolean crawlerInitialized = false;
// Thread objects.
// These get filled in as threads are created.
protected static InitializationThread initializationThread = null;
protected static JobStartThread jobStartThread = null;
protected static StufferThread stufferThread = null;
protected static FinisherThread finisherThread = null;
protected static JobNotificationThread notificationThread = null;
protected static StartupThread startupThread = null;
protected static StartDeleteThread startDeleteThread = null;
protected static JobDeleteThread jobDeleteThread = null;
protected static WorkerThread[] workerThreads = null;
protected static ExpireStufferThread expireStufferThread = null;
protected static ExpireThread[] expireThreads = null;
protected static DocumentDeleteStufferThread deleteStufferThread = null;
protected static DocumentDeleteThread[] deleteThreads = null;
protected static DocumentCleanupStufferThread cleanupStufferThread = null;
protected static DocumentCleanupThread[] cleanupThreads = null;
protected static JobResetThread jobResetThread = null;
protected static SeedingThread seedingThread = null;
protected static IdleCleanupThread idleCleanupThread = null;
protected static SetPriorityThread setPriorityThread = null;
// Reset managers
/** Worker thread pool reset manager */
protected static WorkerResetManager workerResetManager = null;
/** Delete thread pool reset manager */
protected static DocDeleteResetManager docDeleteResetManager = null;
/** Cleanup thread pool reset manager */
protected static DocCleanupResetManager docCleanupResetManager = null;
// Number of worker threads
protected static int numWorkerThreads = 0;
// Number of delete threads
protected static int numDeleteThreads = 0;
// Number of cleanup threads
protected static int numCleanupThreads = 0;
// Number of expiration threads
protected static int numExpireThreads = 0;
// Factor for low water level in queueing
protected static float lowWaterFactor = 5.0f;
// Factor in amount to stuff
protected static float stuffAmtFactor = 0.5f;
// Properties
protected static final String workerThreadCountProperty = "org.apache.manifoldcf.crawler.threads";
protected static final String deleteThreadCountProperty = "org.apache.manifoldcf.crawler.deletethreads";
protected static final String cleanupThreadCountProperty = "org.apache.manifoldcf.crawler.cleanupthreads";
protected static final String expireThreadCountProperty = "org.apache.manifoldcf.crawler.expirethreads";
protected static final String lowWaterFactorProperty = "org.apache.manifoldcf.crawler.lowwaterfactor";
protected static final String stuffAmtFactorProperty = "org.apache.manifoldcf.crawler.stuffamountfactor";
protected static final String connectorsConfigurationFileProperty = "org.apache.manifoldcf.connectorsconfigurationfile";
protected static final String databaseSuperuserNameProperty = "org.apache.manifoldcf.dbsuperusername";
protected static final String databaseSuperuserPasswordProperty = "org.apache.manifoldcf.dbsuperuserpassword";
protected static final String salt = "org.apache.manifoldcf.salt";
/** This object is used to make sure the initialization sequence is atomic. Shutdown cannot occur until the system is in a known state. */
protected static Integer startupLock = new Integer(0);
/** Initialize environment.
*/
public static void initializeEnvironment()
throws ManifoldCFException
{
synchronized (initializeFlagLock)
{
org.apache.manifoldcf.agents.system.ManifoldCF.initializeEnvironment();
org.apache.manifoldcf.authorities.system.ManifoldCF.localInitialize();
org.apache.manifoldcf.crawler.system.ManifoldCF.localInitialize();
}
}
public static void cleanUpEnvironment()
{
synchronized (initializeFlagLock)
{
org.apache.manifoldcf.authorities.system.ManifoldCF.localCleanup();
org.apache.manifoldcf.crawler.system.ManifoldCF.localCleanup();
org.apache.manifoldcf.agents.system.ManifoldCF.cleanUpEnvironment();
}
}
public static void localInitialize()
throws ManifoldCFException
{
synchronized (initializeFlagLock)
{
if (crawlerInitialized)
return;
Logging.initializeLoggers();
Logging.setLogLevels();
crawlerInitialized = true;
}
}
public static void localCleanup()
{
}
/** Create system database using superuser properties from properties.xml.
*/
public static void createSystemDatabase(IThreadContext threadContext)
throws ManifoldCFException
{
// Get the specified superuser name and password, in case this isn't Derby we're using
String superuserName = getProperty(databaseSuperuserNameProperty);
if (superuserName == null)
superuserName = "";
String superuserPassword = getProperty(databaseSuperuserPasswordProperty);
if (superuserPassword == null)
superuserPassword = "";
createSystemDatabase(threadContext,superuserName,superuserPassword);
}
/** Register this agent */
public static void registerThisAgent(IThreadContext tc)
throws ManifoldCFException
{
// Register
IAgentManager agentMgr = AgentManagerFactory.make(tc);
agentMgr.registerAgent("org.apache.manifoldcf.crawler.system.CrawlerAgent");
}
/** Register or re-register all connectors, based on a connectors.xml file.
*/
public static void reregisterAllConnectors(IThreadContext tc)
throws ManifoldCFException
{
// Read connectors configuration file (to figure out what we need to register)
File connectorConfigFile = getFileProperty(connectorsConfigurationFileProperty);
Connectors c = readConnectorDeclarations(connectorConfigFile);
// Unregister all the connectors we don't want.
unregisterAllConnectors(tc,c);
// Register (or update) all connectors specified by connectors.xml
registerConnectors(tc,c);
}
/** Read connectors configuration file.
*/
public static Connectors readConnectorDeclarations(File connectorConfigFile)
throws ManifoldCFException
{
Connectors c = null;
if (connectorConfigFile != null)
{
try
{
// Open the file, read it, and attempt to do the connector registrations
InputStream is = new FileInputStream(connectorConfigFile);
try
{
c = new Connectors(is);
}
finally
{
is.close();
}
}
catch (FileNotFoundException e)
{
throw new ManifoldCFException("Couldn't find connector configuration file: "+e.getMessage(),e);
}
catch (IOException e)
{
throw new ManifoldCFException("Error reading connector configuration file: "+e.getMessage(),e);
}
}
return c;
}
/** Unregister all connectors.
*/
public static void unregisterAllConnectors(IThreadContext tc)
throws ManifoldCFException
{
unregisterAllConnectors(tc,null);
}
// Connectors configuration file
protected static final String NODE_OUTPUTCONNECTOR = "outputconnector";
protected static final String NODE_MAPPINGCONNECTOR = "mappingconnector";
protected static final String NODE_AUTHORITYCONNECTOR = "authorityconnector";
protected static final String NODE_REPOSITORYCONNECTOR = "repositoryconnector";
protected static final String ATTRIBUTE_NAME = "name";
protected static final String ATTRIBUTE_CLASS = "class";
/** Unregister all connectors which don't match a specified connector list.
*/
public static void unregisterAllConnectors(IThreadContext tc, Connectors c)
throws ManifoldCFException
{
// Create a map of class name and description, so we can compare what we can find
// against what we want.
Map<String,String> desiredOutputConnectors = new HashMap<String,String>();
Map<String,String> desiredMappingConnectors = new HashMap<String,String>();
Map<String,String> desiredAuthorityConnectors = new HashMap<String,String>();
Map<String,String> desiredRepositoryConnectors = new HashMap<String,String>();
if (c != null)
{
for (int i = 0; i < c.getChildCount(); i++)
{
ConfigurationNode cn = c.findChild(i);
if (cn.getType().equals(NODE_OUTPUTCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
desiredOutputConnectors.put(className,name);
}
else if (cn.getType().equals(NODE_MAPPINGCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
desiredMappingConnectors.put(className,name);
}
else if (cn.getType().equals(NODE_AUTHORITYCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
desiredAuthorityConnectors.put(className,name);
}
else if (cn.getType().equals(NODE_REPOSITORYCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
desiredRepositoryConnectors.put(className,name);
}
}
}
// Grab a database handle, so we can use transactions later.
IDBInterface database = DBInterfaceFactory.make(tc,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
// Output connectors...
{
IOutputConnectorManager mgr = OutputConnectorManagerFactory.make(tc);
IOutputConnectionManager connManager = OutputConnectionManagerFactory.make(tc);
IResultSet classNames = mgr.getConnectors();
int i = 0;
while (i < classNames.getRowCount())
{
IResultRow row = classNames.getRow(i++);
String className = (String)row.getValue("classname");
String description = (String)row.getValue("description");
if (desiredOutputConnectors.get(className) == null || !desiredOutputConnectors.get(className).equals(description))
{
// Deregistration should be done in a transaction
database.beginTransaction();
try
{
// Find the connection names that come with this class
String[] connectionNames = connManager.findConnectionsForConnector(className);
// For all connection names, notify all agents of the deregistration
AgentManagerFactory.noteOutputConnectorDeregistration(tc,connectionNames);
// Now that all jobs have been placed into an appropriate state, actually do the deregistration itself.
mgr.unregisterConnector(className);
}
catch (ManifoldCFException e)
{
database.signalRollback();
throw e;
}
catch (Error e)
{
database.signalRollback();
throw e;
}
finally
{
database.endTransaction();
}
}
}
System.err.println("Successfully unregistered all output connectors");
}
// Mapping connectors...
{
IMappingConnectorManager mgr = MappingConnectorManagerFactory.make(tc);
IResultSet classNames = mgr.getConnectors();
int i = 0;
while (i < classNames.getRowCount())
{
IResultRow row = classNames.getRow(i++);
String className = (String)row.getValue("classname");
String description = (String)row.getValue("description");
if (desiredMappingConnectors.get(className) == null || !desiredMappingConnectors.get(className).equals(description))
{
mgr.unregisterConnector(className);
}
}
System.err.println("Successfully unregistered all mapping connectors");
}
// Authority connectors...
{
IAuthorityConnectorManager mgr = AuthorityConnectorManagerFactory.make(tc);
IResultSet classNames = mgr.getConnectors();
int i = 0;
while (i < classNames.getRowCount())
{
IResultRow row = classNames.getRow(i++);
String className = (String)row.getValue("classname");
String description = (String)row.getValue("description");
if (desiredAuthorityConnectors.get(className) == null || !desiredAuthorityConnectors.get(className).equals(description))
{
mgr.unregisterConnector(className);
}
}
System.err.println("Successfully unregistered all authority connectors");
}
// Repository connectors...
{
IConnectorManager mgr = ConnectorManagerFactory.make(tc);
IJobManager jobManager = JobManagerFactory.make(tc);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
IResultSet classNames = mgr.getConnectors();
int i = 0;
while (i < classNames.getRowCount())
{
IResultRow row = classNames.getRow(i++);
String className = (String)row.getValue("classname");
String description = (String)row.getValue("description");
if (desiredRepositoryConnectors.get(className) == null || !desiredRepositoryConnectors.get(className).equals(description))
{
// Deregistration should be done in a transaction
database.beginTransaction();
try
{
// Find the connection names that come with this class
String[] connectionNames = connManager.findConnectionsForConnector(className);
// For each connection name, modify the jobs to note that the connector is no longer installed
jobManager.noteConnectorDeregistration(connectionNames);
// Now that all jobs have been placed into an appropriate state, actually do the deregistration itself.
mgr.unregisterConnector(className);
}
catch (ManifoldCFException e)
{
database.signalRollback();
throw e;
}
catch (Error e)
{
database.signalRollback();
throw e;
}
finally
{
database.endTransaction();
}
}
}
System.err.println("Successfully unregistered all repository connectors");
}
}
/** Register all connectors as specified by a Connectors structure, usually read from the connectors.xml file.
*/
public static void registerConnectors(IThreadContext tc, Connectors c)
throws ManifoldCFException
{
if (c != null)
{
// Grab a database handle, so we can use transactions later.
IDBInterface database = DBInterfaceFactory.make(tc,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
// Other code will go here to discover and register various connectors that exist in the classpath
int i = 0;
while (i < c.getChildCount())
{
ConfigurationNode cn = c.findChild(i++);
if (cn.getType().equals(NODE_OUTPUTCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
IOutputConnectorManager mgr = OutputConnectorManagerFactory.make(tc);
IOutputConnectionManager connManager = OutputConnectionManagerFactory.make(tc);
// Registration should be done in a transaction
database.beginTransaction();
try
{
// First, register connector
mgr.registerConnector(name,className);
// Then, signal to all jobs that might depend on this connector that they can switch state
// Find the connection names that come with this class
String[] connectionNames = connManager.findConnectionsForConnector(className);
// For all connection names, notify all agents of the registration
AgentManagerFactory.noteOutputConnectorRegistration(tc,connectionNames);
}
catch (ManifoldCFException e)
{
database.signalRollback();
throw e;
}
catch (Error e)
{
database.signalRollback();
throw e;
}
finally
{
database.endTransaction();
}
System.err.println("Successfully registered output connector '"+className+"'");
}
else if (cn.getType().equals(NODE_AUTHORITYCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
IAuthorityConnectorManager mgr = AuthorityConnectorManagerFactory.make(tc);
mgr.registerConnector(name,className);
System.err.println("Successfully registered authority connector '"+className+"'");
}
else if (cn.getType().equals(NODE_MAPPINGCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
IMappingConnectorManager mgr = MappingConnectorManagerFactory.make(tc);
mgr.registerConnector(name,className);
System.err.println("Successfully registered mapping connector '"+className+"'");
}
else if (cn.getType().equals(NODE_REPOSITORYCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
IConnectorManager mgr = ConnectorManagerFactory.make(tc);
IJobManager jobManager = JobManagerFactory.make(tc);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
// Deregistration should be done in a transaction
database.beginTransaction();
try
{
// First, register connector
mgr.registerConnector(name,className);
// Then, signal to all jobs that might depend on this connector that they can switch state
// Find the connection names that come with this class
String[] connectionNames = connManager.findConnectionsForConnector(className);
// For each connection name, modify the jobs to note that the connector is now installed
jobManager.noteConnectorRegistration(connectionNames);
}
catch (ManifoldCFException e)
{
database.signalRollback();
throw e;
}
catch (Error e)
{
database.signalRollback();
throw e;
}
finally
{
database.endTransaction();
}
System.err.println("Successfully registered repository connector '"+className+"'");
}
else
throw new ManifoldCFException("Unrecognized connectors node type '"+cn.getType()+"'");
}
}
}
/** Install all the crawler system tables.
*@param threadcontext is the thread context.
*/
public static void installSystemTables(IThreadContext threadcontext)
throws ManifoldCFException
{
IConnectorManager repConnMgr = ConnectorManagerFactory.make(threadcontext);
IRepositoryConnectionManager repCon = RepositoryConnectionManagerFactory.make(threadcontext);
IJobManager jobManager = JobManagerFactory.make(threadcontext);
org.apache.manifoldcf.authorities.system.ManifoldCF.installSystemTables(threadcontext);
repConnMgr.install();
repCon.install();
jobManager.install();
}
/** Uninstall all the crawler system tables.
*@param threadcontext is the thread context.
*/
public static void deinstallSystemTables(IThreadContext threadcontext)
throws ManifoldCFException
{
ManifoldCFException se = null;
IConnectorManager repConnMgr = ConnectorManagerFactory.make(threadcontext);
IRepositoryConnectionManager repCon = RepositoryConnectionManagerFactory.make(threadcontext);
IJobManager jobManager = JobManagerFactory.make(threadcontext);
jobManager.deinstall();
repCon.deinstall();
repConnMgr.deinstall();
org.apache.manifoldcf.authorities.system.ManifoldCF.deinstallSystemTables(threadcontext);
if (se != null)
throw se;
}
/** Start everything.
*/
public static void startSystem(IThreadContext threadContext)
throws ManifoldCFException
{
Logging.root.info("Starting up pull-agent...");
synchronized (startupLock)
{
// Now, start all the threads
String maxThreads = getProperty(workerThreadCountProperty);
if (maxThreads == null)
maxThreads = "100";
numWorkerThreads = new Integer(maxThreads).intValue();
if (numWorkerThreads < 1 || numWorkerThreads > 300)
throw new ManifoldCFException("Illegal value for the number of worker threads");
String maxDeleteThreads = getProperty(deleteThreadCountProperty);
if (maxDeleteThreads == null)
maxDeleteThreads = "10";
String maxCleanupThreads = getProperty(cleanupThreadCountProperty);
if (maxCleanupThreads == null)
maxCleanupThreads = "10";
String maxExpireThreads = getProperty(expireThreadCountProperty);
if (maxExpireThreads == null)
maxExpireThreads = "10";
numDeleteThreads = new Integer(maxDeleteThreads).intValue();
if (numDeleteThreads < 1 || numDeleteThreads > 300)
throw new ManifoldCFException("Illegal value for the number of delete threads");
numCleanupThreads = new Integer(maxCleanupThreads).intValue();
if (numCleanupThreads < 1 || numCleanupThreads > 300)
throw new ManifoldCFException("Illegal value for the number of cleanup threads");
numExpireThreads = new Integer(maxExpireThreads).intValue();
if (numExpireThreads < 1 || numExpireThreads > 300)
throw new ManifoldCFException("Illegal value for the number of expire threads");
String lowWaterFactorString = getProperty(lowWaterFactorProperty);
if (lowWaterFactorString == null)
lowWaterFactorString = "5";
lowWaterFactor = new Float(lowWaterFactorString).floatValue();
if (lowWaterFactor < 1.0 || lowWaterFactor > 1000.0)
throw new ManifoldCFException("Illegal value for the low water factor");
String stuffAmtFactorString = getProperty(stuffAmtFactorProperty);
if (stuffAmtFactorString == null)
stuffAmtFactorString = "2";
stuffAmtFactor = new Float(stuffAmtFactorString).floatValue();
if (stuffAmtFactor < 0.1 || stuffAmtFactor > 1000.0)
throw new ManifoldCFException("Illegal value for the stuffing amount factor");
// Create the threads and objects. This MUST be completed before there is any chance of "shutdownSystem" getting called.
QueueTracker queueTracker = new QueueTracker();
DocumentQueue documentQueue = new DocumentQueue();
DocumentDeleteQueue documentDeleteQueue = new DocumentDeleteQueue();
DocumentCleanupQueue documentCleanupQueue = new DocumentCleanupQueue();
DocumentCleanupQueue expireQueue = new DocumentCleanupQueue();
BlockingDocuments blockingDocuments = new BlockingDocuments();
workerResetManager = new WorkerResetManager(documentQueue,expireQueue);
docDeleteResetManager = new DocDeleteResetManager(documentDeleteQueue);
docCleanupResetManager = new DocCleanupResetManager(documentCleanupQueue);
jobStartThread = new JobStartThread();
startupThread = new StartupThread(queueTracker);
startDeleteThread = new StartDeleteThread();
finisherThread = new FinisherThread();
notificationThread = new JobNotificationThread();
jobDeleteThread = new JobDeleteThread();
stufferThread = new StufferThread(documentQueue,numWorkerThreads,workerResetManager,queueTracker,blockingDocuments,lowWaterFactor,stuffAmtFactor);
expireStufferThread = new ExpireStufferThread(expireQueue,numExpireThreads,workerResetManager);
setPriorityThread = new SetPriorityThread(queueTracker,numWorkerThreads,blockingDocuments);
workerThreads = new WorkerThread[numWorkerThreads];
int i = 0;
while (i < numWorkerThreads)
{
workerThreads[i] = new WorkerThread(Integer.toString(i),documentQueue,workerResetManager,queueTracker);
i++;
}
expireThreads = new ExpireThread[numExpireThreads];
i = 0;
while (i < numExpireThreads)
{
expireThreads[i] = new ExpireThread(Integer.toString(i),expireQueue,queueTracker,workerResetManager);
i++;
}
deleteStufferThread = new DocumentDeleteStufferThread(documentDeleteQueue,numDeleteThreads,docDeleteResetManager);
deleteThreads = new DocumentDeleteThread[numDeleteThreads];
i = 0;
while (i < numDeleteThreads)
{
deleteThreads[i] = new DocumentDeleteThread(Integer.toString(i),documentDeleteQueue,docDeleteResetManager);
i++;
}
cleanupStufferThread = new DocumentCleanupStufferThread(documentCleanupQueue,numCleanupThreads,docCleanupResetManager);
cleanupThreads = new DocumentCleanupThread[numCleanupThreads];
i = 0;
while (i < numCleanupThreads)
{
cleanupThreads[i] = new DocumentCleanupThread(Integer.toString(i),documentCleanupQueue,queueTracker,docCleanupResetManager);
i++;
}
jobResetThread = new JobResetThread(queueTracker);
seedingThread = new SeedingThread(queueTracker);
idleCleanupThread = new IdleCleanupThread();
initializationThread = new InitializationThread(queueTracker);
// Start the initialization thread. This does the initialization work and starts all the other threads when that's done. It then exits.
initializationThread.start();
}
Logging.root.info("Pull-agent started");
}
protected static class InitializationThread extends Thread
{
protected final QueueTracker queueTracker;
public InitializationThread(QueueTracker queueTracker)
{
super();
this.queueTracker = queueTracker;
setName("Initialization thread");
setDaemon(true);
}
public void run()
{
int i;
// Initialize the database
try
{
IThreadContext threadContext = ThreadContextFactory.make();
// First, get a job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(threadContext);
Logging.threads.debug("Agents process starting initialization...");
// Call the database to get it ready
jobManager.prepareForStart();
Logging.threads.debug("Agents process reprioritizing documents...");
HashMap connectionMap = new HashMap();
HashMap jobDescriptionMap = new HashMap();
// Reprioritize all documents in the jobqueue, 1000 at a time
long currentTime = System.currentTimeMillis();
// Do the 'not yet processed' documents only. Documents that are queued for reprocessing will be assigned
// new priorities. Already processed documents won't. This guarantees that our bins are appropriate for current thread
// activity.
// In order for this to be the correct functionality, ALL reseeding and requeuing operations MUST reset the associated document
// priorities.
while (true)
{
long startTime = System.currentTimeMillis();
DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(currentTime, 10000);
if (docs.length == 0)
break;
// Calculate new priorities for all these documents
writeDocumentPriorities(threadContext,mgr,jobManager,docs,connectionMap,jobDescriptionMap,queueTracker,currentTime);
Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
}
Logging.threads.debug("Agents process initialization complete!");
// Start all the threads
jobStartThread.start();
startupThread.start();
startDeleteThread.start();
finisherThread.start();
notificationThread.start();
jobDeleteThread.start();
stufferThread.start();
expireStufferThread.start();
setPriorityThread.start();
i = 0;
while (i < numWorkerThreads)
{
workerThreads[i].start();
i++;
}
i = 0;
while (i < numExpireThreads)
{
expireThreads[i].start();
i++;
}
cleanupStufferThread.start();
i = 0;
while (i < numCleanupThreads)
{
cleanupThreads[i].start();
i++;
}
deleteStufferThread.start();
i = 0;
while (i < numDeleteThreads)
{
deleteThreads[i].start();
i++;
}
jobResetThread.start();
seedingThread.start();
idleCleanupThread.start();
// exit!
}
catch (Throwable e)
{
// Severe error on initialization
if (e instanceof ManifoldCFException)
{
// Deal with interrupted exception gracefully, because it means somebody is trying to shut us down before we got started.
if (((ManifoldCFException)e).getErrorCode() == ManifoldCFException.INTERRUPTED)
return;
}
System.err.println("agents process could not start - shutting down");
Logging.threads.fatal("Startup initialization error tossed: "+e.getMessage(),e);
System.exit(-300);
}
}
}
/** Stop the system.
*/
public static void stopSystem(IThreadContext threadContext)
throws ManifoldCFException
{
Logging.root.info("Shutting down pull-agent...");
synchronized (startupLock)
{
while (initializationThread != null || jobDeleteThread != null || startupThread != null || startDeleteThread != null ||
jobStartThread != null || stufferThread != null ||
finisherThread != null || notificationThread != null || workerThreads != null || expireStufferThread != null || expireThreads != null ||
deleteStufferThread != null || deleteThreads != null ||
cleanupStufferThread != null || cleanupThreads != null ||
jobResetThread != null || seedingThread != null || idleCleanupThread != null || setPriorityThread != null)
{
// Send an interrupt to all threads that are still there.
// In theory, this only needs to be done once. In practice, I have seen cases where the thread loses track of the fact that it has been
// interrupted (which may be a JVM bug - who knows?), but in any case there's no harm in doing it again.
if (initializationThread != null)
{
initializationThread.interrupt();
}
if (setPriorityThread != null)
{
setPriorityThread.interrupt();
}
if (jobStartThread != null)
{
jobStartThread.interrupt();
}
if (jobDeleteThread != null)
{
jobDeleteThread.interrupt();
}
if (startupThread != null)
{
startupThread.interrupt();
}
if (startDeleteThread != null)
{
startDeleteThread.interrupt();
}
if (stufferThread != null)
{
stufferThread.interrupt();
}
if (expireStufferThread != null)
{
expireStufferThread.interrupt();
}
if (finisherThread != null)
{
finisherThread.interrupt();
}
if (notificationThread != null)
{
notificationThread.interrupt();
}
if (workerThreads != null)
{
int i = 0;
while (i < workerThreads.length)
{
Thread workerThread = workerThreads[i++];
if (workerThread != null)
workerThread.interrupt();
}
}
if (expireThreads != null)
{
int i = 0;
while (i < expireThreads.length)
{
Thread expireThread = expireThreads[i++];
if (expireThread != null)
expireThread.interrupt();
}
}
if (cleanupStufferThread != null)
{
cleanupStufferThread.interrupt();
}
if (cleanupThreads != null)
{
int i = 0;
while (i < cleanupThreads.length)
{
Thread cleanupThread = cleanupThreads[i++];
if (cleanupThread != null)
cleanupThread.interrupt();
}
}
if (deleteStufferThread != null)
{
deleteStufferThread.interrupt();
}
if (deleteThreads != null)
{
int i = 0;
while (i < deleteThreads.length)
{
Thread deleteThread = deleteThreads[i++];
if (deleteThread != null)
deleteThread.interrupt();
}
}
if (jobResetThread != null)
{
jobResetThread.interrupt();
}
if (seedingThread != null)
{
seedingThread.interrupt();
}
if (idleCleanupThread != null)
{
idleCleanupThread.interrupt();
}
// Now, wait for all threads to die.
try
{
ManifoldCF.sleep(1000L);
}
catch (InterruptedException e)
{
}
// Check to see which died.
if (initializationThread != null)
{
if (!initializationThread.isAlive())
initializationThread = null;
}
if (setPriorityThread != null)
{
if (!setPriorityThread.isAlive())
setPriorityThread = null;
}
if (jobDeleteThread != null)
{
if (!jobDeleteThread.isAlive())
jobDeleteThread = null;
}
if (startupThread != null)
{
if (!startupThread.isAlive())
startupThread = null;
}
if (startDeleteThread != null)
{
if (!startDeleteThread.isAlive())
startDeleteThread = null;
}
if (jobStartThread != null)
{
if (!jobStartThread.isAlive())
jobStartThread = null;
}
if (stufferThread != null)
{
if (!stufferThread.isAlive())
stufferThread = null;
}
if (expireStufferThread != null)
{
if (!expireStufferThread.isAlive())
expireStufferThread = null;
}
if (finisherThread != null)
{
if (!finisherThread.isAlive())
finisherThread = null;
}
if (notificationThread != null)
{
if (!notificationThread.isAlive())
notificationThread = null;
}
if (workerThreads != null)
{
int i = 0;
boolean isAlive = false;
while (i < workerThreads.length)
{
Thread workerThread = workerThreads[i];
if (workerThread != null)
{
if (!workerThread.isAlive())
workerThreads[i] = null;
else
isAlive = true;
}
i++;
}
if (!isAlive)
workerThreads = null;
}
if (expireThreads != null)
{
int i = 0;
boolean isAlive = false;
while (i < expireThreads.length)
{
Thread expireThread = expireThreads[i];
if (expireThread != null)
{
if (!expireThread.isAlive())
expireThreads[i] = null;
else
isAlive = true;
}
i++;
}
if (!isAlive)
expireThreads = null;
}
if (cleanupStufferThread != null)
{
if (!cleanupStufferThread.isAlive())
cleanupStufferThread = null;
}
if (cleanupThreads != null)
{
int i = 0;
boolean isAlive = false;
while (i < cleanupThreads.length)
{
Thread cleanupThread = cleanupThreads[i];
if (cleanupThread != null)
{
if (!cleanupThread.isAlive())
cleanupThreads[i] = null;
else
isAlive = true;
}
i++;
}
if (!isAlive)
cleanupThreads = null;
}
if (deleteStufferThread != null)
{
if (!deleteStufferThread.isAlive())
deleteStufferThread = null;
}
if (deleteThreads != null)
{
int i = 0;
boolean isAlive = false;
while (i < deleteThreads.length)
{
Thread deleteThread = deleteThreads[i];
if (deleteThread != null)
{
if (!deleteThread.isAlive())
deleteThreads[i] = null;
else
isAlive = true;
}
i++;
}
if (!isAlive)
deleteThreads = null;
}
if (jobResetThread != null)
{
if (!jobResetThread.isAlive())
jobResetThread = null;
}
if (seedingThread != null)
{
if (!seedingThread.isAlive())
seedingThread = null;
}
if (idleCleanupThread != null)
{
if (!idleCleanupThread.isAlive())
idleCleanupThread = null;
}
}
// Threads are down; release connectors
RepositoryConnectorFactory.closeAllConnectors(threadContext);
numWorkerThreads = 0;
numDeleteThreads = 0;
numExpireThreads = 0;
}
Logging.root.info("Pull-agent successfully shut down");
}
/** Atomically export the crawler configuration */
public static void exportConfiguration(IThreadContext threadContext, String exportFilename, String passCode)
throws ManifoldCFException
{
// The basic idea here is that we open a zip stream, into which we dump all the pertinent information in a transactionally-consistent manner.
// First, we need a database handle...
IDBInterface database = DBInterfaceFactory.make(threadContext,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
// Also create the following managers, which will handle the actual details of writing configuration data
IOutputConnectionManager outputManager = OutputConnectionManagerFactory.make(threadContext);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(threadContext);
IMappingConnectionManager mappingManager = MappingConnectionManagerFactory.make(threadContext);
IAuthorityConnectionManager authManager = AuthorityConnectionManagerFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
File outputFile = new File(exportFilename);
// Create a zip output stream, which is what we will use as a mechanism for handling the output data
try
{
OutputStream os = new FileOutputStream(outputFile);
try
{
java.util.zip.ZipOutputStream zos = null;
CipherOutputStream cos = null;
// Check whether we need to encrypt the file content:
if (passCode != null && passCode.length() > 0)
{
// Write IV as a prefix:
SecureRandom random = new SecureRandom();
byte[] iv = new byte[IV_LENGTH];
random.nextBytes(iv);
os.write(iv);
os.flush();
Cipher cipher = null;
try
{
cipher = getCipher(Cipher.ENCRYPT_MODE, passCode, iv);
}
catch (GeneralSecurityException gse)
{
throw new ManifoldCFException("Could not encrypt configuratiom file: " + gse.getMessage());
}
cos = new CipherOutputStream(os, cipher);
zos = new java.util.zip.ZipOutputStream(cos);
}
else
zos = new java.util.zip.ZipOutputStream(os);
try
{
// Now, work within a transaction.
database.beginTransaction();
try
{
// At the outermost level, I've decided that the best structure is to have a zipentry for each
// manager. Each manager must manage its own data as a binary blob, including any format versioning information,
// This guarantees flexibility for the future.
// The zipentries must be written in an order that permits their proper restoration. The "lowest level" is thus
// written first, which yields the order: authority connections, repository connections, jobs
java.util.zip.ZipEntry outputEntry = new java.util.zip.ZipEntry("outputs");
zos.putNextEntry(outputEntry);
outputManager.exportConfiguration(zos);
zos.closeEntry();
java.util.zip.ZipEntry mappingEntry = new java.util.zip.ZipEntry("mappings");
zos.putNextEntry(mappingEntry);
mappingManager.exportConfiguration(zos);
zos.closeEntry();
java.util.zip.ZipEntry authEntry = new java.util.zip.ZipEntry("authorities");
zos.putNextEntry(authEntry);
authManager.exportConfiguration(zos);
zos.closeEntry();
java.util.zip.ZipEntry connEntry = new java.util.zip.ZipEntry("connections");
zos.putNextEntry(connEntry);
connManager.exportConfiguration(zos);
zos.closeEntry();
java.util.zip.ZipEntry jobsEntry = new java.util.zip.ZipEntry("jobs");
zos.putNextEntry(jobsEntry);
jobManager.exportConfiguration(zos);
zos.closeEntry();
// All done
}
catch (ManifoldCFException e)
{
database.signalRollback();
throw e;
}
catch (Error e)
{
database.signalRollback();
throw e;
}
finally
{
database.endTransaction();
}
}
finally
{
zos.close();
if (cos != null) {
cos.close();
}
}
}
finally
{
os.close();
}
}
catch (java.io.IOException e)
{
// On error, delete any file we created
outputFile.delete();
// Convert I/O error into lcf exception
throw new ManifoldCFException("Error creating configuration file: "+e.getMessage(),e);
}
}
/** Atomically import a crawler configuration */
public static void importConfiguration(IThreadContext threadContext, String importFilename, String passCode)
throws ManifoldCFException
{
// First, we need a database handle...
IDBInterface database = DBInterfaceFactory.make(threadContext,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
// Also create the following managers, which will handle the actual details of reading configuration data
IOutputConnectionManager outputManager = OutputConnectionManagerFactory.make(threadContext);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(threadContext);
IMappingConnectionManager mappingManager = MappingConnectionManagerFactory.make(threadContext);
IAuthorityConnectionManager authManager = AuthorityConnectionManagerFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
File inputFile = new File(importFilename);
// Create a zip input stream, which is what we will use as a mechanism for handling the input data
try
{
InputStream is = new FileInputStream(inputFile);
try
{
java.util.zip.ZipInputStream zis = null;
CipherInputStream cis = null;
// Check whether we need to decrypt the file content:
if (passCode != null && passCode.length() > 0)
{
byte[] iv = new byte[IV_LENGTH];
is.read(iv);
Cipher cipher = null;
try
{
cipher = getCipher(Cipher.DECRYPT_MODE, passCode, iv);
}
catch (GeneralSecurityException gse)
{
throw new ManifoldCFException("Could not decrypt configuratiom file: " + gse.getMessage());
}
cis = new CipherInputStream(is, cipher);
zis = new java.util.zip.ZipInputStream(cis);
}
else
zis = new java.util.zip.ZipInputStream(is);
try
{
// Now, work within a transaction.
database.beginTransaction();
try
{
// Process the entries in the order in which they were recorded.
int entries = 0;
while (true)
{
java.util.zip.ZipEntry z = zis.getNextEntry();
// Stop if there are no more entries
if (z == null)
break;
entries++;
// Get the name of the entry
String name = z.getName();
if (name.equals("outputs"))
outputManager.importConfiguration(zis);
else if (name.equals("mappings"))
mappingManager.importConfiguration(zis);
else if (name.equals("authorities"))
authManager.importConfiguration(zis);
else if (name.equals("connections"))
connManager.importConfiguration(zis);
else if (name.equals("jobs"))
jobManager.importConfiguration(zis);
else
throw new ManifoldCFException("Configuration file has an entry named '"+name+"' that I do not recognize");
zis.closeEntry();
}
if (entries == 0 && passCode != null && passCode.length() > 0)
throw new ManifoldCFException("Cannot read configuration file. Please check your passcode and/or SALT value.");
// All done!!
}
catch (ManifoldCFException e)
{
database.signalRollback();
throw e;
}
catch (Error e)
{
database.signalRollback();
throw e;
}
finally
{
database.endTransaction();
}
}
finally
{
zis.close();
if (cis != null) {
cis.close();
}
}
}
finally
{
is.close();
}
}
catch (java.io.IOException e)
{
// Convert I/O error into lcf exception
throw new ManifoldCFException("Error reading configuration file: "+e.getMessage(),e);
}
}
/** Get the maximum number of worker threads.
*/
public static int getMaxWorkerThreads()
{
return numWorkerThreads;
}
/** Get the maximum number of delete threads.
*/
public static int getMaxDeleteThreads()
{
return numDeleteThreads;
}
/** Get the maximum number of expire threads.
*/
public static int getMaxExpireThreads()
{
return numExpireThreads;
}
/** Requeue documents due to carrydown.
*/
public static void requeueDocumentsDueToCarrydown(IJobManager jobManager, DocumentDescription[] requeueCandidates,
IRepositoryConnector connector, IRepositoryConnection connection, QueueTracker queueTracker, long currentTime)
throws ManifoldCFException
{
// A list of document descriptions from finishDocuments() above represents those documents that may need to be requeued, for the
// reason that carrydown information for those documents has changed. In order to requeue, we need to calculate document priorities, however.
double[] docPriorities = new double[requeueCandidates.length];
String[][] binNames = new String[requeueCandidates.length][];
int q = 0;
while (q < requeueCandidates.length)
{
DocumentDescription dd = requeueCandidates[q];
String[] bins = calculateBins(connector,dd.getDocumentIdentifier());
binNames[q] = bins;
docPriorities[q] = queueTracker.calculatePriority(bins,connection);
if (Logging.scheduling.isDebugEnabled())
Logging.scheduling.debug("Document '"+dd.getDocumentIdentifier()+" given priority "+new Double(docPriorities[q]).toString());
q++;
}
// Now, requeue the documents with the new priorities
boolean[] trackerNote = jobManager.carrydownChangeDocumentMultiple(requeueCandidates,currentTime,docPriorities);
// Free the unused priorities.
// Inform queuetracker about what we used and what we didn't
q = 0;
while (q < trackerNote.length)
{
if (trackerNote[q] == false)
{
String[] bins = binNames[q];
queueTracker.notePriorityNotUsed(bins,connection,docPriorities[q]);
}
q++;
}
}
/** Stuff colons so we can't have conflicts. */
public static String colonStuff(String input)
{
StringBuilder sb = new StringBuilder();
int i = 0;
while (i < input.length())
{
char x = input.charAt(i++);
if (x == ':' || x == '\\')
sb.append('\\');
sb.append(x);
}
return sb.toString();
}
/** Create a global string */
public static String createGlobalString(String simpleString)
{
return ":" + simpleString;
}
/** Create a connection-specific string */
public static String createConnectionSpecificString(String connectionName, String simpleString)
{
return "C "+colonStuff(connectionName) + ":" + simpleString;
}
/** Create a job-specific string */
public static String createJobSpecificString(Long jobID, String simpleString)
{
return "J "+jobID.toString() + ":" + simpleString;
}
/** Given a connector object and a document identifier, calculate its bins.
*/
public static String[] calculateBins(IRepositoryConnector connector, String documentIdentifier)
{
// Get the bins for the document identifier
return connector.getBinNames(documentIdentifier);
}
/** Reset all (active) document priorities. This operation may occur due to various externally-triggered
* events, such a job abort, pause, resume, wait, or unwait.
*/
public static void resetAllDocumentPriorities(IThreadContext threadContext, QueueTracker queueTracker, long currentTime)
throws ManifoldCFException
{
IJobManager jobManager = JobManagerFactory.make(threadContext);
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
// Reset the queue tracker
queueTracker.beginReset();
// Perform the reprioritization, for all active documents in active jobs. During this time,
// it is safe to have other threads assign new priorities to documents, but it is NOT safe
// for other threads to attempt to change the minimum priority level. The queuetracker object
// will therefore block that from occurring, until the reset is complete.
try
{
// Reprioritize all documents in the jobqueue, 1000 at a time
HashMap connectionMap = new HashMap();
HashMap jobDescriptionMap = new HashMap();
// Do the 'not yet processed' documents only. Documents that are queued for reprocessing will be assigned
// new priorities. Already processed documents won't. This guarantees that our bins are appropriate for current thread
// activity.
// In order for this to be the correct functionality, ALL reseeding and requeuing operations MUST reset the associated document
// priorities.
while (true)
{
long startTime = System.currentTimeMillis();
DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(currentTime, 10000);
if (docs.length == 0)
break;
// Calculate new priorities for all these documents
writeDocumentPriorities(threadContext,connectionManager,jobManager,docs,connectionMap,jobDescriptionMap,queueTracker,currentTime);
Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
}
}
finally
{
queueTracker.endReset();
}
}
/** Write a set of document priorities, based on the current queue tracker.
*/
public static void writeDocumentPriorities(IThreadContext threadContext, IRepositoryConnectionManager mgr, IJobManager jobManager, DocumentDescription[] descs, HashMap connectionMap, HashMap jobDescriptionMap, QueueTracker queueTracker, long currentTime)
throws ManifoldCFException
{
if (Logging.scheduling.isDebugEnabled())
Logging.scheduling.debug("Reprioritizing "+Integer.toString(descs.length)+" documents");
double[] priorities = new double[descs.length];
// Go through the documents and calculate the priorities
int i = 0;
while (i < descs.length)
{
DocumentDescription dd = descs[i];
IJobDescription job = (IJobDescription)jobDescriptionMap.get(dd.getJobID());
if (job == null)
{
job = jobManager.load(dd.getJobID(),true);
jobDescriptionMap.put(dd.getJobID(),job);
}
String connectionName = job.getConnectionName();
IRepositoryConnection connection = (IRepositoryConnection)connectionMap.get(connectionName);
if (connection == null)
{
connection = mgr.load(connectionName);
connectionMap.put(connectionName,connection);
}
String[] binNames;
// Grab a connector handle
IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
connection.getClassName(),
connection.getConfigParams(),
connection.getMaxConnections());
try
{
if (connector == null)
binNames = new String[]{""};
else
// Get the bins for the document identifier
binNames = connector.getBinNames(descs[i].getDocumentIdentifier());
}
finally
{
RepositoryConnectorFactory.release(connector);
}
priorities[i] = queueTracker.calculatePriority(binNames,connection);
if (Logging.scheduling.isDebugEnabled())
Logging.scheduling.debug("Document '"+dd.getDocumentIdentifier()+"' given priority "+new Double(priorities[i]).toString());
i++;
}
// Now, write all the priorities we can.
jobManager.writeDocumentPriorities(currentTime,descs,priorities);
}
/** Request permission from agent to delete an output connection.
*@param threadContext is the thread context.
*@param connName is the name of the output connection.
*@return true if the connection is in use, false otherwise.
*/
public static boolean isOutputConnectionInUse(IThreadContext threadContext, String connName)
throws ManifoldCFException
{
// Check with job manager.
IJobManager jobManager = JobManagerFactory.make(threadContext);
return jobManager.checkIfOutputReference(connName);
}
/** Note the deregistration of a set of output connections.
*@param threadContext is the thread context.
*@param connectionNames are the names of the connections being deregistered.
*/
public static void noteOutputConnectorDeregistration(IThreadContext threadContext, String[] connectionNames)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteOutputConnectorDeregistration(connectionNames);
}
/** Note the registration of a set of output connections.
*@param threadContext is the thread context.
*@param connectionNames are the names of the connections being registered.
*/
public static void noteOutputConnectorRegistration(IThreadContext threadContext, String[] connectionNames)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteOutputConnectorRegistration(connectionNames);
}
/** Note the change in configuration of an output connection.
*@param threadContext is the thread context.
*@param connectionName is the output connection name.
*/
public static void noteOutputConnectionChange(IThreadContext threadContext, String connectionName)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteOutputConnectionChange(connectionName);
}
/** Qualify output activity name.
*@param outputActivityName is the name of the output activity.
*@param outputConnectionName is the corresponding name of the output connection.
*@return the qualified (global) activity name.
*/
public static String qualifyOutputActivityName(String outputActivityName, String outputConnectionName)
{
return outputActivityName+" ("+outputConnectionName+")";
}
/** Get the activities list for a given repository connection.
*/
public static String[] getActivitiesList(IThreadContext threadContext, String connectionName)
throws ManifoldCFException
{
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
IRepositoryConnection thisConnection = connectionManager.load(connectionName);
if (thisConnection == null)
return null;
String[] outputActivityList = OutputConnectionManagerFactory.getAllOutputActivities(threadContext);
String[] connectorActivityList = RepositoryConnectorFactory.getActivitiesList(threadContext,thisConnection.getClassName());
String[] globalActivityList = IRepositoryConnectionManager.activitySet;
String[] activityList = new String[outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
int k2 = 0;
int j;
if (outputActivityList != null)
{
j = 0;
while (j < outputActivityList.length)
{
activityList[k2++] = outputActivityList[j++];
}
}
if (connectorActivityList != null)
{
j = 0;
while (j < connectorActivityList.length)
{
activityList[k2++] = connectorActivityList[j++];
}
}
j = 0;
while (j < globalActivityList.length)
{
activityList[k2++] = globalActivityList[j++];
}
java.util.Arrays.sort(activityList);
return activityList;
}
private static final int IV_LENGTH = 16;
private static Cipher getCipher(final int mode, final String passCode, final byte[] iv) throws GeneralSecurityException,
ManifoldCFException
{
final String saltValue = getProperty(salt);
if (saltValue == null || saltValue.length() == 0)
throw new ManifoldCFException("Missing required SALT value");
SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
KeySpec keySpec = new PBEKeySpec(passCode.toCharArray(), saltValue.getBytes(), 1024, 128);
SecretKey secretKey = factory.generateSecret(keySpec);
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
SecretKeySpec key = new SecretKeySpec(secretKey.getEncoded(), "AES");
IvParameterSpec parameterSpec = new IvParameterSpec(iv);
cipher.init(mode, key, parameterSpec);
return cipher;
}
// ========================== API support ===========================
protected static final String API_JOBNODE = "job";
protected static final String API_JOBSTATUSNODE = "jobstatus";
protected static final String API_REPOSITORYCONNECTORNODE = "repositoryconnector";
protected static final String API_OUTPUTCONNECTORNODE = "outputconnector";
protected static final String API_AUTHORITYCONNECTORNODE = "authorityconnector";
protected static final String API_MAPPINGCONNECTORNODE = "mappingconnector";
protected static final String API_REPOSITORYCONNECTIONNODE = "repositoryconnection";
protected static final String API_OUTPUTCONNECTIONNODE = "outputconnection";
protected static final String API_AUTHORITYCONNECTIONNODE = "authorityconnection";
protected static final String API_MAPPINGCONNECTIONNODE = "mappingconnection";
protected static final String API_CHECKRESULTNODE = "check_result";
protected static final String API_JOBIDNODE = "job_id";
protected static final String API_CONNECTIONNAMENODE = "connection_name";
protected final static String API_ROWNODE = "row";
protected final static String API_COLUMNNODE = "column";
protected final static String API_NAMENODE = "name";
protected final static String API_VALUENODE = "value";
protected final static String API_ACTIVITYNODE = "activity";
// Connector nodes
protected static final String CONNECTORNODE_DESCRIPTION = "description";
protected static final String CONNECTORNODE_CLASSNAME = "class_name";
/** Decode path element.
* Path elements in the API world cannot have "/" characters, or they become impossible to parse. This method undoes
* escaping that prevents "/" from appearing.
*/
public static String decodeAPIPathElement(String startingPathElement)
throws ManifoldCFException
{
StringBuilder sb = new StringBuilder();
int i = 0;
while (i < startingPathElement.length())
{
char x = startingPathElement.charAt(i++);
if (x == '.')
{
if (i == startingPathElement.length())
throw new ManifoldCFException("Element decoding failed; illegal '.' character in '"+startingPathElement+"'");
x = startingPathElement.charAt(i++);
if (x == '.')
sb.append(x);
else if (x == '+')
sb.append('/');
else
throw new ManifoldCFException("Element decoding failed; illegal post-'.' character in '"+startingPathElement+"'");
}
else
sb.append(x);
}
return sb.toString();
}
// Read (GET) functions
// Read result codes
public static final int READRESULT_NOTFOUND = 0;
public static final int READRESULT_FOUND = 1;
/** Read jobs */
protected static int apiReadJobs(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
IJobDescription[] jobs = jobManager.getAllJobs();
int i = 0;
while (i < jobs.length)
{
ConfigurationNode jobNode = new ConfigurationNode(API_JOBNODE);
formatJobDescription(jobNode,jobs[i++]);
output.addChild(output.getChildCount(),jobNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read a job */
protected static int apiReadJob(IThreadContext tc, Configuration output, Long jobID)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
IJobDescription job = jobManager.load(jobID);
if (job != null)
{
// Fill the return object with job information
ConfigurationNode jobNode = new ConfigurationNode(API_JOBNODE);
formatJobDescription(jobNode,job);
output.addChild(output.getChildCount(),jobNode);
}
else
{
createErrorNode(output,"Job does not exist.");
return READRESULT_NOTFOUND;
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read an output connection status */
protected static int apiReadOutputConnectionStatus(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
IOutputConnection connection = connectionManager.load(connectionName);
if (connection == null)
{
createErrorNode(output,"Connection '"+connectionName+"' does not exist");
return READRESULT_NOTFOUND;
}
String results;
// Grab a connection handle, and call the test method
IOutputConnector connector = OutputConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
try
{
results = connector.check();
}
catch (ManifoldCFException e)
{
results = e.getMessage();
}
finally
{
OutputConnectorFactory.release(connector);
}
ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
response.setValue(results);
output.addChild(output.getChildCount(),response);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read an authority connection status */
protected static int apiReadAuthorityConnectionStatus(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IAuthorityConnectionManager connectionManager = AuthorityConnectionManagerFactory.make(tc);
IAuthorityConnection connection = connectionManager.load(connectionName);
if (connection == null)
{
createErrorNode(output,"Connection '"+connectionName+"' does not exist");
return READRESULT_NOTFOUND;
}
String results;
// Grab a connection handle, and call the test method
IAuthorityConnector connector = AuthorityConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
try
{
results = connector.check();
}
catch (ManifoldCFException e)
{
results = e.getMessage();
}
finally
{
AuthorityConnectorFactory.release(connector);
}
ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
response.setValue(results);
output.addChild(output.getChildCount(),response);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read a mapping connection status */
protected static int apiReadMappingConnectionStatus(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IMappingConnectionManager connectionManager = MappingConnectionManagerFactory.make(tc);
IMappingConnection connection = connectionManager.load(connectionName);
if (connection == null)
{
createErrorNode(output,"Connection '"+connectionName+"' does not exist");
return READRESULT_NOTFOUND;
}
String results;
// Grab a connection handle, and call the test method
IMappingConnector connector = MappingConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
try
{
results = connector.check();
}
catch (ManifoldCFException e)
{
results = e.getMessage();
}
finally
{
MappingConnectorFactory.release(connector);
}
ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
response.setValue(results);
output.addChild(output.getChildCount(),response);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read a repository connection status */
protected static int apiReadRepositoryConnectionStatus(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
IRepositoryConnection connection = connectionManager.load(connectionName);
if (connection == null)
{
createErrorNode(output,"Connection '"+connectionName+"' does not exist");
return READRESULT_NOTFOUND;
}
String results;
// Grab a connection handle, and call the test method
IRepositoryConnector connector = RepositoryConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
try
{
results = connector.check();
}
catch (ManifoldCFException e)
{
results = e.getMessage();
}
finally
{
RepositoryConnectorFactory.release(connector);
}
ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
response.setValue(results);
output.addChild(output.getChildCount(),response);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read an output connection's info */
protected static int apiReadOutputConnectionInfo(IThreadContext tc, Configuration output, String connectionName, String command)
throws ManifoldCFException
{
try
{
IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
IOutputConnection connection = connectionManager.load(connectionName);
if (connection == null)
{
createErrorNode(output,"Connection '"+connectionName+"' does not exist");
return READRESULT_NOTFOUND;
}
// Grab a connection handle, and call the test method
IOutputConnector connector = OutputConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
try
{
return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND;
}
finally
{
OutputConnectorFactory.release(connector);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read a repository connection's info */
protected static int apiReadRepositoryConnectionInfo(IThreadContext tc, Configuration output, String connectionName, String command)
throws ManifoldCFException
{
try
{
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
IRepositoryConnection connection = connectionManager.load(connectionName);
if (connection == null)
{
createErrorNode(output,"Connection '"+connectionName+"' does not exist");
return READRESULT_NOTFOUND;
}
// Grab a connection handle, and call the test method
IRepositoryConnector connector = RepositoryConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
try
{
return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND;
}
finally
{
RepositoryConnectorFactory.release(connector);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Get api job statuses */
protected static int apiReadJobStatuses(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
JobStatus[] jobStatuses = jobManager.getAllStatus();
int i = 0;
while (i < jobStatuses.length)
{
ConfigurationNode jobStatusNode = new ConfigurationNode(API_JOBSTATUSNODE);
formatJobStatus(jobStatusNode,jobStatuses[i++]);
output.addChild(output.getChildCount(),jobStatusNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Get api job status */
protected static int apiReadJobStatus(IThreadContext tc, Configuration output, Long jobID)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
JobStatus status = jobManager.getStatus(jobID);
if (status != null)
{
ConfigurationNode jobStatusNode = new ConfigurationNode(API_JOBSTATUSNODE);
formatJobStatus(jobStatusNode,status);
output.addChild(output.getChildCount(),jobStatusNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Get api job status with no counts */
protected static int apiReadJobStatusNoCounts(IThreadContext tc, Configuration output, Long jobID)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
JobStatus status = jobManager.getStatus(jobID,false);
if (status != null)
{
ConfigurationNode jobStatusNode = new ConfigurationNode(API_JOBSTATUSNODE);
formatJobStatus(jobStatusNode,status);
output.addChild(output.getChildCount(),jobStatusNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Get output connections */
protected static int apiReadOutputConnections(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
try
{
IOutputConnectionManager connManager = OutputConnectionManagerFactory.make(tc);
IOutputConnection[] connections = connManager.getAllConnections();
int i = 0;
while (i < connections.length)
{
ConfigurationNode connectionNode = new ConfigurationNode(API_OUTPUTCONNECTIONNODE);
formatOutputConnection(connectionNode,connections[i++]);
output.addChild(output.getChildCount(),connectionNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read output connection */
protected static int apiReadOutputConnection(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
IOutputConnection connection = connectionManager.load(connectionName);
if (connection != null)
{
// Fill the return object with job information
ConfigurationNode connectionNode = new ConfigurationNode(API_OUTPUTCONNECTIONNODE);
formatOutputConnection(connectionNode,connection);
output.addChild(output.getChildCount(),connectionNode);
}
else
{
createErrorNode(output,"Connection '"+connectionName+"' does not exist.");
return READRESULT_NOTFOUND;
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Get authority connections */
protected static int apiReadAuthorityConnections(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
try
{
IAuthorityConnectionManager connManager = AuthorityConnectionManagerFactory.make(tc);
IAuthorityConnection[] connections = connManager.getAllConnections();
int i = 0;
while (i < connections.length)
{
ConfigurationNode connectionNode = new ConfigurationNode(API_AUTHORITYCONNECTIONNODE);
formatAuthorityConnection(connectionNode,connections[i++]);
output.addChild(output.getChildCount(),connectionNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Get mapping connections */
protected static int apiReadMappingConnections(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
try
{
IMappingConnectionManager connManager = MappingConnectionManagerFactory.make(tc);
IMappingConnection[] connections = connManager.getAllConnections();
int i = 0;
while (i < connections.length)
{
ConfigurationNode connectionNode = new ConfigurationNode(API_MAPPINGCONNECTIONNODE);
formatMappingConnection(connectionNode,connections[i++]);
output.addChild(output.getChildCount(),connectionNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read authority connection */
protected static int apiReadAuthorityConnection(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IAuthorityConnectionManager connectionManager = AuthorityConnectionManagerFactory.make(tc);
IAuthorityConnection connection = connectionManager.load(connectionName);
if (connection != null)
{
// Fill the return object with job information
ConfigurationNode connectionNode = new ConfigurationNode(API_AUTHORITYCONNECTIONNODE);
formatAuthorityConnection(connectionNode,connection);
output.addChild(output.getChildCount(),connectionNode);
}
else
{
createErrorNode(output,"Authority connection '"+connectionName+"' does not exist.");
return READRESULT_NOTFOUND;
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read mapping connection */
protected static int apiReadMappingConnection(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IMappingConnectionManager connectionManager = MappingConnectionManagerFactory.make(tc);
IMappingConnection connection = connectionManager.load(connectionName);
if (connection != null)
{
// Fill the return object with job information
ConfigurationNode connectionNode = new ConfigurationNode(API_MAPPINGCONNECTIONNODE);
formatMappingConnection(connectionNode,connection);
output.addChild(output.getChildCount(),connectionNode);
}
else
{
createErrorNode(output,"Mapping connection '"+connectionName+"' does not exist.");
return READRESULT_NOTFOUND;
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Get repository connections */
protected static int apiReadRepositoryConnections(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
try
{
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
IRepositoryConnection[] connections = connManager.getAllConnections();
int i = 0;
while (i < connections.length)
{
ConfigurationNode connectionNode = new ConfigurationNode(API_REPOSITORYCONNECTIONNODE);
formatRepositoryConnection(connectionNode,connections[i++]);
output.addChild(output.getChildCount(),connectionNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Read repository connection */
protected static int apiReadRepositoryConnection(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
IRepositoryConnection connection = connectionManager.load(connectionName);
if (connection != null)
{
// Fill the return object with job information
ConfigurationNode connectionNode = new ConfigurationNode(API_REPOSITORYCONNECTIONNODE);
formatRepositoryConnection(connectionNode,connection);
output.addChild(output.getChildCount(),connectionNode);
}
else
{
createErrorNode(output,"Repository connection '"+connectionName+"' does not exist");
return READRESULT_NOTFOUND;
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** List output connectors */
protected static int apiReadOutputConnectors(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
// List registered output connectors
try
{
IOutputConnectorManager manager = OutputConnectorManagerFactory.make(tc);
IResultSet resultSet = manager.getConnectors();
int j = 0;
while (j < resultSet.getRowCount())
{
IResultRow row = resultSet.getRow(j++);
ConfigurationNode child = new ConfigurationNode(API_OUTPUTCONNECTORNODE);
String description = (String)row.getValue("description");
String className = (String)row.getValue("classname");
ConfigurationNode node;
if (description != null)
{
node = new ConfigurationNode(CONNECTORNODE_DESCRIPTION);
node.setValue(description);
child.addChild(child.getChildCount(),node);
}
node = new ConfigurationNode(CONNECTORNODE_CLASSNAME);
node.setValue(className);
child.addChild(child.getChildCount(),node);
output.addChild(output.getChildCount(),child);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** List authority connectors */
protected static int apiReadAuthorityConnectors(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
// List registered authority connectors
try
{
IAuthorityConnectorManager manager = AuthorityConnectorManagerFactory.make(tc);
IResultSet resultSet = manager.getConnectors();
int j = 0;
while (j < resultSet.getRowCount())
{
IResultRow row = resultSet.getRow(j++);
ConfigurationNode child = new ConfigurationNode(API_AUTHORITYCONNECTORNODE);
String description = (String)row.getValue("description");
String className = (String)row.getValue("classname");
ConfigurationNode node;
if (description != null)
{
node = new ConfigurationNode(CONNECTORNODE_DESCRIPTION);
node.setValue(description);
child.addChild(child.getChildCount(),node);
}
node = new ConfigurationNode(CONNECTORNODE_CLASSNAME);
node.setValue(className);
child.addChild(child.getChildCount(),node);
output.addChild(output.getChildCount(),child);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** List mapping connectors */
protected static int apiReadMappingConnectors(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
// List registered authority connectors
try
{
IMappingConnectorManager manager = MappingConnectorManagerFactory.make(tc);
IResultSet resultSet = manager.getConnectors();
int j = 0;
while (j < resultSet.getRowCount())
{
IResultRow row = resultSet.getRow(j++);
ConfigurationNode child = new ConfigurationNode(API_MAPPINGCONNECTORNODE);
String description = (String)row.getValue("description");
String className = (String)row.getValue("classname");
ConfigurationNode node;
if (description != null)
{
node = new ConfigurationNode(CONNECTORNODE_DESCRIPTION);
node.setValue(description);
child.addChild(child.getChildCount(),node);
}
node = new ConfigurationNode(CONNECTORNODE_CLASSNAME);
node.setValue(className);
child.addChild(child.getChildCount(),node);
output.addChild(output.getChildCount(),child);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** List repository connectors */
protected static int apiReadRepositoryConnectors(IThreadContext tc, Configuration output)
throws ManifoldCFException
{
// List registered repository connectors
try
{
IConnectorManager manager = ConnectorManagerFactory.make(tc);
IResultSet resultSet = manager.getConnectors();
int j = 0;
while (j < resultSet.getRowCount())
{
IResultRow row = resultSet.getRow(j++);
ConfigurationNode child = new ConfigurationNode(API_REPOSITORYCONNECTORNODE);
String description = (String)row.getValue("description");
String className = (String)row.getValue("classname");
ConfigurationNode node;
if (description != null)
{
node = new ConfigurationNode(CONNECTORNODE_DESCRIPTION);
node.setValue(description);
child.addChild(child.getChildCount(),node);
}
node = new ConfigurationNode(CONNECTORNODE_CLASSNAME);
node.setValue(className);
child.addChild(child.getChildCount(),node);
output.addChild(output.getChildCount(),child);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
protected final static Map<String,Integer> docState;
static
{
docState = new HashMap<String,Integer>();
docState.put("neverprocessed",new Integer(IJobManager.DOCSTATE_NEVERPROCESSED));
docState.put("previouslyprocessed",new Integer(IJobManager.DOCSTATE_PREVIOUSLYPROCESSED));
docState.put("outofscope",new Integer(IJobManager.DOCSTATE_OUTOFSCOPE));
}
protected final static Map<String,Integer> docStatus;
static
{
docStatus = new HashMap<String,Integer>();
docStatus.put("inactive",new Integer(IJobManager.DOCSTATUS_INACTIVE));
docStatus.put("processing",new Integer(IJobManager.DOCSTATUS_PROCESSING));
docStatus.put("expiring",new Integer(IJobManager.DOCSTATUS_EXPIRING));
docStatus.put("deleting",new Integer(IJobManager.DOCSTATUS_DELETING));
docStatus.put("readyforprocessing",new Integer(IJobManager.DOCSTATUS_READYFORPROCESSING));
docStatus.put("readyforexpiration",new Integer(IJobManager.DOCSTATUS_READYFOREXPIRATION));
docStatus.put("waitingforprocessing",new Integer(IJobManager.DOCSTATUS_WAITINGFORPROCESSING));
docStatus.put("waitingforexpiration",new Integer(IJobManager.DOCSTATUS_WAITINGFOREXPIRATION));
docStatus.put("waitingforever",new Integer(IJobManager.DOCSTATUS_WAITINGFOREVER));
docStatus.put("hopcountexceeded",new Integer(IJobManager.DOCSTATUS_HOPCOUNTEXCEEDED));
}
/** Queue reports */
protected static int apiReadRepositoryConnectionQueue(IThreadContext tc, Configuration output,
String connectionName, Map<String,List<String>> queryParameters) throws ManifoldCFException
{
if (queryParameters == null)
queryParameters = new HashMap<String,List<String>>();
// Jobs (specified by id)
Long[] jobs;
List<String> jobList = queryParameters.get("job");
if (jobList == null)
jobs = new Long[0];
else
{
jobs = new Long[jobList.size()];
for (int i = 0; i < jobs.length; i++)
{
jobs[i] = new Long(jobList.get(i));
}
}
// Now time
long now;
List<String> nowList = queryParameters.get("now");
if (nowList == null || nowList.size() == 0)
now = System.currentTimeMillis();
else if (nowList.size() > 1)
throw new ManifoldCFException("Multiple values for now parameter");
else
now = new Long(nowList.get(0)).longValue();
// Identifier match
RegExpCriteria idMatch;
List<String> idMatchList = queryParameters.get("idmatch");
List<String> idMatchInsensitiveList = queryParameters.get("idmatch_insensitive");
if (idMatchList != null && idMatchInsensitiveList != null)
throw new ManifoldCFException("Either use idmatch or idmatch_insensitive, not both.");
boolean isInsensitiveIdMatch;
if (idMatchInsensitiveList != null)
{
idMatchList = idMatchInsensitiveList;
isInsensitiveIdMatch = true;
}
else
isInsensitiveIdMatch = false;
if (idMatchList == null || idMatchList.size() == 0)
idMatch = null;
else if (idMatchList.size() > 1)
throw new ManifoldCFException("Multiple id match regexps specified.");
else
idMatch = new RegExpCriteria(idMatchList.get(0),isInsensitiveIdMatch);
List<String> stateMatchList = queryParameters.get("statematch");
int[] matchStates;
if (stateMatchList == null)
matchStates = new int[0];
else
{
matchStates = new int[stateMatchList.size()];
for (int i = 0; i < matchStates.length; i++)
{
Integer value = docState.get(stateMatchList.get(i));
if (value == null)
throw new ManifoldCFException("Unrecognized state value: '"+stateMatchList.get(i)+"'");
matchStates[i] = value.intValue();
}
}
List<String> statusMatchList = queryParameters.get("statusmatch");
int[] matchStatuses;
if (statusMatchList == null)
matchStatuses = new int[0];
else
{
matchStatuses = new int[statusMatchList.size()];
for (int i = 0; i < matchStatuses.length; i++)
{
Integer value = docStatus.get(statusMatchList.get(i));
if (value == null)
throw new ManifoldCFException("Unrecognized status value: '"+statusMatchList.get(i)+"'");
matchStatuses[i] = value.intValue();
}
}
StatusFilterCriteria filterCriteria = new StatusFilterCriteria(jobs,now,idMatch,matchStates,matchStatuses);
// Look for sort order parameters...
SortOrder sortOrder = new SortOrder();
List<String> sortColumnsList = queryParameters.get("sortcolumn");
List<String> sortColumnsDirList = queryParameters.get("sortcolumn_direction");
if (sortColumnsList != null || sortColumnsDirList != null)
{
if (sortColumnsList == null || sortColumnsDirList == null || sortColumnsList.size() != sortColumnsDirList.size())
throw new ManifoldCFException("sortcolumn and sortcolumn_direction must have the same cardinality.");
for (int i = 0; i < sortColumnsList.size(); i++)
{
String column = sortColumnsList.get(i);
String dir = sortColumnsDirList.get(i);
int dirInt;
if (dir.equals("ascending"))
dirInt = SortOrder.SORT_ASCENDING;
else if (dir.equals("descending"))
dirInt = SortOrder.SORT_DESCENDING;
else
throw new ManifoldCFException("sortcolumn_direction must be 'ascending' or 'descending'.");
sortOrder.addCriteria(column,dirInt);
}
}
// Start row and row count
int startRow;
List<String> startRowList = queryParameters.get("startrow");
if (startRowList == null || startRowList.size() == 0)
startRow = 0;
else if (startRowList.size() > 1)
throw new ManifoldCFException("Multiple start rows specified.");
else
startRow = new Integer(startRowList.get(0)).intValue();
int rowCount;
List<String> rowCountList = queryParameters.get("rowcount");
if (rowCountList == null || rowCountList.size() == 0)
rowCount = 20;
else if (rowCountList.size() > 1)
throw new ManifoldCFException("Multiple row counts specified.");
else
rowCount = new Integer(rowCountList.get(0)).intValue();
List<String> reportTypeList = queryParameters.get("report");
String reportType;
if (reportTypeList == null || reportTypeList.size() == 0)
reportType = "document";
else if (reportTypeList.size() > 1)
throw new ManifoldCFException("Multiple report types specified.");
else
reportType = reportTypeList.get(0);
IJobManager jobManager = JobManagerFactory.make(tc);
IResultSet result;
String[] resultColumns;
if (reportType.equals("document"))
{
try
{
result = jobManager.genDocumentStatus(connectionName,filterCriteria,sortOrder,startRow,rowCount);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
return READRESULT_FOUND;
}
resultColumns = new String[]{"identifier","job","state","status","scheduled","action","retrycount","retrylimit"};
}
else if (reportType.equals("status"))
{
BucketDescription idBucket;
List<String> idBucketList = queryParameters.get("idbucket");
List<String> idBucketInsensitiveList = queryParameters.get("idbucket_insensitive");
if (idBucketList != null && idBucketInsensitiveList != null)
throw new ManifoldCFException("Either use idbucket or idbucket_insensitive, not both.");
boolean isInsensitiveIdBucket;
if (idBucketInsensitiveList != null)
{
idBucketList = idBucketInsensitiveList;
isInsensitiveIdBucket = true;
}
else
isInsensitiveIdBucket = false;
if (idBucketList == null || idBucketList.size() == 0)
idBucket = new BucketDescription("()",false);
else if (idBucketList.size() > 1)
throw new ManifoldCFException("Multiple idbucket regexps specified.");
else
idBucket = new BucketDescription(idBucketList.get(0),isInsensitiveIdBucket);
try
{
result = jobManager.genQueueStatus(connectionName,filterCriteria,sortOrder,idBucket,startRow,rowCount);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
return READRESULT_FOUND;
}
resultColumns = new String[]{"idbucket","inactive","processing","expiring","deleting",
"processready","expireready","processwaiting","expirewaiting","waitingforever","hopcountexceeded"};
}
else
throw new ManifoldCFException("Unknown report type '"+reportType+"'.");
createResultsetNode(output,result,resultColumns);
return READRESULT_FOUND;
}
/** Get jobs for connection */
protected static int apiReadRepositoryConnectionJobs(IThreadContext tc, Configuration output,
String connectionName) throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
IJobDescription[] jobs = jobManager.findJobsForConnection(connectionName);
if (jobs == null)
{
createErrorNode(output,"Unknown connection '"+connectionName+"'");
return READRESULT_NOTFOUND;
}
int i = 0;
while (i < jobs.length)
{
ConfigurationNode jobNode = new ConfigurationNode(API_JOBNODE);
formatJobDescription(jobNode,jobs[i++]);
output.addChild(output.getChildCount(),jobNode);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** History reports */
protected static int apiReadRepositoryConnectionHistory(IThreadContext tc, Configuration output,
String connectionName, Map<String,List<String>> queryParameters) throws ManifoldCFException
{
if (queryParameters == null)
queryParameters = new HashMap<String,List<String>>();
// Look for filter criteria parameters...
// Start time
List<String> startTimeList = queryParameters.get("starttime");
Long startTime;
if (startTimeList == null || startTimeList.size() == 0)
startTime = null;
else if (startTimeList.size() > 1)
throw new ManifoldCFException("Multiple start times specified.");
else
startTime = new Long(startTimeList.get(0));
// End time
List<String> endTimeList = queryParameters.get("endtime");
Long endTime;
if (endTimeList == null || endTimeList.size() == 0)
endTime = null;
else if (endTimeList.size() > 1)
throw new ManifoldCFException("Multiple end times specified.");
else
endTime = new Long(endTimeList.get(0));
// Activities
List<String> activityList = queryParameters.get("activity");
String[] activities;
if (activityList == null)
activities = new String[0];
else
activities = activityList.toArray(new String[0]);
// Entity match
RegExpCriteria entityMatch;
List<String> entityMatchList = queryParameters.get("entitymatch");
List<String> entityMatchInsensitiveList = queryParameters.get("entitymatch_insensitive");
if (entityMatchList != null && entityMatchInsensitiveList != null)
throw new ManifoldCFException("Either use entitymatch or entitymatch_insensitive, not both.");
boolean isInsensitiveEntityMatch;
if (entityMatchInsensitiveList != null)
{
entityMatchList = entityMatchInsensitiveList;
isInsensitiveEntityMatch = true;
}
else
isInsensitiveEntityMatch = false;
if (entityMatchList == null || entityMatchList.size() == 0)
entityMatch = null;
else if (entityMatchList.size() > 1)
throw new ManifoldCFException("Multiple entity match regexps specified.");
else
entityMatch = new RegExpCriteria(entityMatchList.get(0),isInsensitiveEntityMatch);
// Result code match
RegExpCriteria resultCodeMatch;
List<String> resultCodeMatchList = queryParameters.get("resultcodematch");
List<String> resultCodeMatchInsensitiveList = queryParameters.get("resultcodematch_insensitive");
if (resultCodeMatchList != null && resultCodeMatchInsensitiveList != null)
throw new ManifoldCFException("Either use resultcodematch or resultcodematch_insensitive, not both.");
boolean isInsensitiveResultCodeMatch;
if (entityMatchInsensitiveList != null)
{
resultCodeMatchList = resultCodeMatchInsensitiveList;
isInsensitiveResultCodeMatch = true;
}
else
isInsensitiveResultCodeMatch = false;
if (resultCodeMatchList == null || resultCodeMatchList.size() == 0)
resultCodeMatch = null;
else if (resultCodeMatchList.size() > 1)
throw new ManifoldCFException("Multiple resultcode match regexps specified.");
else
resultCodeMatch = new RegExpCriteria(resultCodeMatchList.get(0),isInsensitiveResultCodeMatch);
// Filter criteria
FilterCriteria filterCriteria = new FilterCriteria(activities,startTime,endTime,entityMatch,resultCodeMatch);
// Look for sort order parameters...
SortOrder sortOrder = new SortOrder();
List<String> sortColumnsList = queryParameters.get("sortcolumn");
List<String> sortColumnsDirList = queryParameters.get("sortcolumn_direction");
if (sortColumnsList != null || sortColumnsDirList != null)
{
if (sortColumnsList == null || sortColumnsDirList == null || sortColumnsList.size() != sortColumnsDirList.size())
throw new ManifoldCFException("sortcolumn and sortcolumn_direction must have the same cardinality.");
for (int i = 0; i < sortColumnsList.size(); i++)
{
String column = sortColumnsList.get(i);
String dir = sortColumnsDirList.get(i);
int dirInt;
if (dir.equals("ascending"))
dirInt = SortOrder.SORT_ASCENDING;
else if (dir.equals("descending"))
dirInt = SortOrder.SORT_DESCENDING;
else
throw new ManifoldCFException("sortcolumn_direction must be 'ascending' or 'descending'.");
sortOrder.addCriteria(column,dirInt);
}
}
// Start row and row count
int startRow;
List<String> startRowList = queryParameters.get("startrow");
if (startRowList == null || startRowList.size() == 0)
startRow = 0;
else if (startRowList.size() > 1)
throw new ManifoldCFException("Multiple start rows specified.");
else
startRow = new Integer(startRowList.get(0)).intValue();
int rowCount;
List<String> rowCountList = queryParameters.get("rowcount");
if (rowCountList == null || rowCountList.size() == 0)
rowCount = 20;
else if (rowCountList.size() > 1)
throw new ManifoldCFException("Multiple row counts specified.");
else
rowCount = new Integer(rowCountList.get(0)).intValue();
List<String> reportTypeList = queryParameters.get("report");
String reportType;
if (reportTypeList == null || reportTypeList.size() == 0)
reportType = "simple";
else if (reportTypeList.size() > 1)
throw new ManifoldCFException("Multiple report types specified.");
else
reportType = reportTypeList.get(0);
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
IResultSet result;
String[] resultColumns;
if (reportType.equals("simple"))
{
try
{
result = connectionManager.genHistorySimple(connectionName,filterCriteria,sortOrder,startRow,rowCount);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
return READRESULT_FOUND;
}
resultColumns = new String[]{"starttime","resultcode","resultdesc","identifier","activity","bytes","elapsedtime"};
}
else if (reportType.equals("maxactivity"))
{
long maxInterval = connectionManager.getMaxRows();
long actualRows = connectionManager.countHistoryRows(connectionName,filterCriteria);
if (actualRows > maxInterval)
throw new ManifoldCFException("Too many history rows specified for maxactivity report - actual is "+actualRows+", max is "+maxInterval+".");
BucketDescription idBucket;
List<String> idBucketList = queryParameters.get("idbucket");
List<String> idBucketInsensitiveList = queryParameters.get("idbucket_insensitive");
if (idBucketList != null && idBucketInsensitiveList != null)
throw new ManifoldCFException("Either use idbucket or idbucket_insensitive, not both.");
boolean isInsensitiveIdBucket;
if (idBucketInsensitiveList != null)
{
idBucketList = idBucketInsensitiveList;
isInsensitiveIdBucket = true;
}
else
isInsensitiveIdBucket = false;
if (idBucketList == null || idBucketList.size() == 0)
idBucket = new BucketDescription("()",false);
else if (idBucketList.size() > 1)
throw new ManifoldCFException("Multiple idbucket regexps specified.");
else
idBucket = new BucketDescription(idBucketList.get(0),isInsensitiveIdBucket);
long interval;
List<String> intervalList = queryParameters.get("interval");
if (intervalList == null || intervalList.size() == 0)
interval = 300000L;
else if (intervalList.size() > 1)
throw new ManifoldCFException("Multiple intervals specified.");
else
interval = new Long(intervalList.get(0)).longValue();
try
{
result = connectionManager.genHistoryActivityCount(connectionName,filterCriteria,sortOrder,idBucket,interval,startRow,rowCount);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
return READRESULT_FOUND;
}
resultColumns = new String[]{"starttime","endtime","activitycount","idbucket"};
}
else if (reportType.equals("maxbandwidth"))
{
long maxInterval = connectionManager.getMaxRows();
long actualRows = connectionManager.countHistoryRows(connectionName,filterCriteria);
if (actualRows > maxInterval)
throw new ManifoldCFException("Too many history rows specified for maxbandwidth report - actual is "+actualRows+", max is "+maxInterval+".");
BucketDescription idBucket;
List<String> idBucketList = queryParameters.get("idbucket");
List<String> idBucketInsensitiveList = queryParameters.get("idbucket_insensitive");
if (idBucketList != null && idBucketInsensitiveList != null)
throw new ManifoldCFException("Either use idbucket or idbucket_insensitive, not both.");
boolean isInsensitiveIdBucket;
if (idBucketInsensitiveList != null)
{
idBucketList = idBucketInsensitiveList;
isInsensitiveIdBucket = true;
}
else
isInsensitiveIdBucket = false;
if (idBucketList == null || idBucketList.size() == 0)
idBucket = new BucketDescription("()",false);
else if (idBucketList.size() > 1)
throw new ManifoldCFException("Multiple idbucket regexps specified.");
else
idBucket = new BucketDescription(idBucketList.get(0),isInsensitiveIdBucket);
long interval;
List<String> intervalList = queryParameters.get("interval");
if (intervalList == null || intervalList.size() == 0)
interval = 300000L;
else if (intervalList.size() > 1)
throw new ManifoldCFException("Multiple intervals specified.");
else
interval = new Long(intervalList.get(0)).longValue();
try
{
result = connectionManager.genHistoryByteCount(connectionName,filterCriteria,sortOrder,idBucket,interval,startRow,rowCount);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
return READRESULT_FOUND;
}
resultColumns = new String[]{"starttime","endtime","bytecount","idbucket"};
}
else if (reportType.equals("result"))
{
BucketDescription idBucket;
List<String> idBucketList = queryParameters.get("idbucket");
List<String> idBucketInsensitiveList = queryParameters.get("idbucket_insensitive");
if (idBucketList != null && idBucketInsensitiveList != null)
throw new ManifoldCFException("Either use idbucket or idbucket_insensitive, not both.");
boolean isInsensitiveIdBucket;
if (idBucketInsensitiveList != null)
{
idBucketList = idBucketInsensitiveList;
isInsensitiveIdBucket = true;
}
else
isInsensitiveIdBucket = false;
if (idBucketList == null || idBucketList.size() == 0)
idBucket = new BucketDescription("()",false);
else if (idBucketList.size() > 1)
throw new ManifoldCFException("Multiple idbucket regexps specified.");
else
idBucket = new BucketDescription(idBucketList.get(0),isInsensitiveIdBucket);
BucketDescription resultCodeBucket;
List<String> resultCodeBucketList = queryParameters.get("resultcodebucket");
List<String> resultCodeBucketInsensitiveList = queryParameters.get("resultcodebucket_insensitive");
if (resultCodeBucketList != null && resultCodeBucketInsensitiveList != null)
throw new ManifoldCFException("Either use resultcodebucket or resultcodebucket_insensitive, not both.");
boolean isInsensitiveResultCodeBucket;
if (resultCodeBucketInsensitiveList != null)
{
resultCodeBucketList = resultCodeBucketInsensitiveList;
isInsensitiveResultCodeBucket = true;
}
else
isInsensitiveResultCodeBucket = false;
if (resultCodeBucketList == null || resultCodeBucketList.size() == 0)
resultCodeBucket = new BucketDescription("(.*)",false);
else if (resultCodeBucketList.size() > 1)
throw new ManifoldCFException("Multiple resultcodebucket regexps specified.");
else
resultCodeBucket = new BucketDescription(resultCodeBucketList.get(0),isInsensitiveResultCodeBucket);
try
{
result = connectionManager.genHistoryResultCodes(connectionName,filterCriteria,sortOrder,resultCodeBucket,idBucket,startRow,rowCount);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
return READRESULT_FOUND;
}
resultColumns = new String[]{"idbucket","resultcodebucket","eventcount"};
}
else
throw new ManifoldCFException("Unknown report type '"+reportType+"'.");
createResultsetNode(output,result,resultColumns);
return READRESULT_FOUND;
}
/** Add a resultset node to the output. */
protected static void createResultsetNode(Configuration output, IResultSet result, String[] resultColumns)
throws ManifoldCFException
{
// Go through result set and add results to output
for (int i = 0; i < result.getRowCount(); i++)
{
IResultRow row = result.getRow(i);
ConfigurationNode rowValue = new ConfigurationNode(API_ROWNODE);
for (String columnName : resultColumns)
{
ConfigurationNode columnValue = new ConfigurationNode(API_COLUMNNODE);
Object value = row.getValue(columnName);
String valueToUse;
if (value == null)
valueToUse = "";
else
valueToUse = value.toString();
ConfigurationNode nameNode = new ConfigurationNode(API_NAMENODE);
nameNode.setValue(columnName);
columnValue.addChild(columnValue.getChildCount(),nameNode);
ConfigurationNode valueNode = new ConfigurationNode(API_VALUENODE);
valueNode.setValue(valueToUse);
columnValue.addChild(columnValue.getChildCount(),valueNode);
rowValue.addChild(rowValue.getChildCount(),columnValue);
}
output.addChild(output.getChildCount(),rowValue);
}
}
/** Read the activity list for a given connection name. */
protected static int apiReadRepositoryConnectionActivities(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
String[] activities = getActivitiesList(tc,connectionName);
if (activities == null)
{
createErrorNode(output,"Connection '"+connectionName+"' does not exist.");
return READRESULT_NOTFOUND;
}
for (String activity : activities)
{
ConfigurationNode node = new ConfigurationNode(API_ACTIVITYNODE);
node.setValue(activity);
output.addChild(output.getChildCount(),node);
}
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return READRESULT_FOUND;
}
/** Execute specified read command.
*@param tc is the thread context.
*@param output is the output object, to be filled in.
*@param path is the object path.
*@return read status - either found, not found, or bad args
*/
public static int executeReadCommand(IThreadContext tc, Configuration output, String path,
Map<String,List<String>> queryParameters) throws ManifoldCFException
{
if (path.equals("jobs"))
{
return apiReadJobs(tc,output);
}
else if (path.startsWith("jobs/"))
{
Long jobID = new Long(path.substring("jobs/".length()));
return apiReadJob(tc,output,jobID);
}
else if (path.startsWith("repositoryconnectionactivities/"))
{
int firstSeparator = "repositoryconnectionactivities/".length();
String connectionName = decodeAPIPathElement(path.substring(firstSeparator));
return apiReadRepositoryConnectionActivities(tc,output,connectionName);
}
else if (path.startsWith("repositoryconnectionhistory/"))
{
int firstSeparator = "repositoryconnectionhistory/".length();
String connectionName = decodeAPIPathElement(path.substring(firstSeparator));
return apiReadRepositoryConnectionHistory(tc,output,connectionName,queryParameters);
}
else if (path.startsWith("repositoryconnectionqueue/"))
{
int firstSeparator = "repositoryconnectionqueue/".length();
String connectionName = decodeAPIPathElement(path.substring(firstSeparator));
return apiReadRepositoryConnectionQueue(tc,output,connectionName,queryParameters);
}
else if (path.startsWith("repositoryconnectionjobs/"))
{
int firstSeparator = "repositoryconnectionjobs/".length();
String connectionName = decodeAPIPathElement(path.substring(firstSeparator));
return apiReadRepositoryConnectionJobs(tc,output,connectionName);
}
else if (path.startsWith("status/"))
{
int firstSeparator = "status/".length();
int secondSeparator = path.indexOf("/",firstSeparator);
if (secondSeparator == -1)
{
createErrorNode(output,"Need connection name.");
return READRESULT_NOTFOUND;
}
String connectionType = path.substring(firstSeparator,secondSeparator);
String connectionName = decodeAPIPathElement(path.substring(secondSeparator+1));
if (connectionType.equals("outputconnections"))
{
return apiReadOutputConnectionStatus(tc,output,connectionName);
}
else if (connectionType.equals("mappingconnections"))
{
return apiReadMappingConnectionStatus(tc,output,connectionName);
}
else if (connectionType.equals("authorityconnections"))
{
return apiReadAuthorityConnectionStatus(tc,output,connectionName);
}
else if (connectionType.equals("repositoryconnections"))
{
return apiReadRepositoryConnectionStatus(tc,output,connectionName);
}
else
{
createErrorNode(output,"Unknown connection type '"+connectionType+"'.");
return READRESULT_NOTFOUND;
}
}
else if (path.startsWith("info/"))
{
int firstSeparator = "info/".length();
int secondSeparator = path.indexOf("/",firstSeparator);
if (secondSeparator == -1)
{
createErrorNode(output,"Need connection type and connection name.");
return READRESULT_NOTFOUND;
}
int thirdSeparator = path.indexOf("/",secondSeparator+1);
if (thirdSeparator == -1)
{
createErrorNode(output,"Need connection name.");
return READRESULT_NOTFOUND;
}
String connectionType = path.substring(firstSeparator,secondSeparator);
String connectionName = decodeAPIPathElement(path.substring(secondSeparator+1,thirdSeparator));
String command = path.substring(thirdSeparator+1);
if (connectionType.equals("outputconnections"))
{
return apiReadOutputConnectionInfo(tc,output,connectionName,command);
}
else if (connectionType.equals("repositoryconnections"))
{
return apiReadRepositoryConnectionInfo(tc,output,connectionName,command);
}
else
{
createErrorNode(output,"Unknown connection type '"+connectionType+"'.");
return READRESULT_NOTFOUND;
}
}
else if (path.equals("jobstatuses"))
{
return apiReadJobStatuses(tc,output);
}
else if (path.startsWith("jobstatuses/"))
{
Long jobID = new Long(path.substring("jobstatuses/".length()));
return apiReadJobStatus(tc,output,jobID);
}
else if (path.startsWith("jobstatusesnocounts/"))
{
Long jobID = new Long(path.substring("jobstatusesnocounts/".length()));
return apiReadJobStatusNoCounts(tc,output,jobID);
}
else if (path.equals("outputconnections"))
{
return apiReadOutputConnections(tc,output);
}
else if (path.startsWith("outputconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("outputconnections/".length()));
return apiReadOutputConnection(tc,output,connectionName);
}
else if (path.equals("mappingconnections"))
{
return apiReadMappingConnections(tc,output);
}
else if (path.startsWith("mappingconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("mappingconnections/".length()));
return apiReadMappingConnection(tc,output,connectionName);
}
else if (path.equals("authorityconnections"))
{
return apiReadAuthorityConnections(tc,output);
}
else if (path.startsWith("authorityconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("authorityconnections/".length()));
return apiReadAuthorityConnection(tc,output,connectionName);
}
else if (path.equals("repositoryconnections"))
{
return apiReadRepositoryConnections(tc,output);
}
else if (path.startsWith("repositoryconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("repositoryconnections/".length()));
return apiReadRepositoryConnection(tc,output,connectionName);
}
else if (path.equals("outputconnectors"))
{
return apiReadOutputConnectors(tc,output);
}
else if (path.equals("mappingconnectors"))
{
return apiReadMappingConnectors(tc,output);
}
else if (path.equals("authorityconnectors"))
{
return apiReadAuthorityConnectors(tc,output);
}
else if (path.equals("repositoryconnectors"))
{
return apiReadRepositoryConnectors(tc,output);
}
else
{
createErrorNode(output,"Unrecognized resource.");
return READRESULT_NOTFOUND;
}
}
// Post result codes
public static final int POSTRESULT_NOTFOUND = 0;
public static final int POSTRESULT_FOUND = 1;
public static final int POSTRESULT_CREATED = 2;
/** Post job.
*/
protected static int apiPostJob(IThreadContext tc, Configuration output, Configuration input)
throws ManifoldCFException
{
ConfigurationNode jobNode = findConfigurationNode(input,API_JOBNODE);
if (jobNode == null)
throw new ManifoldCFException("Input must have '"+API_JOBNODE+"' field");
// Turn the configuration node into a JobDescription
org.apache.manifoldcf.crawler.jobs.JobDescription job = new org.apache.manifoldcf.crawler.jobs.JobDescription();
processJobDescription(job,jobNode);
if (job.getID() != null)
throw new ManifoldCFException("Input job cannot supply an ID field for create");
try
{
Long jobID = new Long(IDFactory.make(tc));
job.setID(jobID);
job.setIsNew(true);
// Save the job.
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.save(job);
ConfigurationNode idNode = new ConfigurationNode(API_JOBIDNODE);
idNode.setValue(jobID.toString());
output.addChild(output.getChildCount(),idNode);
return POSTRESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return POSTRESULT_FOUND;
}
/** Execute specified post command.
*@param tc is the thread context.
*@param output is the output object, to be filled in.
*@param path is the object path.
*@param input is the input object.
*@return write result - either "not found", "found", or "created".
*/
public static int executePostCommand(IThreadContext tc, Configuration output, String path, Configuration input)
throws ManifoldCFException
{
if (path.equals("jobs"))
{
return apiPostJob(tc,output,input);
}
else
{
createErrorNode(output,"Unrecognized resource.");
return POSTRESULT_NOTFOUND;
}
}
// Write result codes
public static final int WRITERESULT_NOTFOUND = 0;
public static final int WRITERESULT_FOUND = 1;
public static final int WRITERESULT_CREATED = 2;
/** Start a job.
*/
protected static int apiWriteStartJob(IThreadContext tc, Configuration output, Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.manualStart(jobID,requestMinimum);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Abort a job.
*/
protected static int apiWriteAbortJob(IThreadContext tc, Configuration output, Long jobID)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.manualAbort(jobID);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Restart a job.
*/
protected static int apiWriteRestartJob(IThreadContext tc, Configuration output, Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.manualAbortRestart(jobID,requestMinimum);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Pause a job.
*/
protected static int apiWritePauseJob(IThreadContext tc, Configuration output, Long jobID)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.pauseJob(jobID);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Resume a job.
*/
protected static int apiWriteResumeJob(IThreadContext tc, Configuration output, Long jobID)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.restartJob(jobID);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Write job.
*/
protected static int apiWriteJob(IThreadContext tc, Configuration output, Configuration input, Long jobID)
throws ManifoldCFException
{
ConfigurationNode jobNode = findConfigurationNode(input,API_JOBNODE);
if (jobNode == null)
throw new ManifoldCFException("Input must have '"+API_JOBNODE+"' field");
// Turn the configuration node into a JobDescription
org.apache.manifoldcf.crawler.jobs.JobDescription job = new org.apache.manifoldcf.crawler.jobs.JobDescription();
processJobDescription(job,jobNode);
try
{
if (job.getID() == null)
{
job.setID(jobID);
}
else
{
if (!job.getID().equals(jobID))
throw new ManifoldCFException("Job identifier must agree within object and within path");
}
job.setIsNew(false);
// Save the job.
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.save(job);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Write output connection.
*/
protected static int apiWriteOutputConnection(IThreadContext tc, Configuration output, Configuration input, String connectionName)
throws ManifoldCFException
{
ConfigurationNode connectionNode = findConfigurationNode(input,API_OUTPUTCONNECTIONNODE);
if (connectionNode == null)
throw new ManifoldCFException("Input argument must have '"+API_OUTPUTCONNECTIONNODE+"' field");
// Turn the configuration node into an OutputConnection
org.apache.manifoldcf.agents.outputconnection.OutputConnection outputConnection = new org.apache.manifoldcf.agents.outputconnection.OutputConnection();
processOutputConnection(outputConnection,connectionNode);
if (outputConnection.getName() == null)
outputConnection.setName(connectionName);
else
{
if (!outputConnection.getName().equals(connectionName))
throw new ManifoldCFException("Connection name in path and in object must agree");
}
try
{
// Save the connection.
IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
if (connectionManager.save(outputConnection))
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Write authority connection.
*/
protected static int apiWriteAuthorityConnection(IThreadContext tc, Configuration output, Configuration input, String connectionName)
throws ManifoldCFException
{
ConfigurationNode connectionNode = findConfigurationNode(input,API_AUTHORITYCONNECTIONNODE);
if (connectionNode == null)
throw new ManifoldCFException("Input argument must have '"+API_AUTHORITYCONNECTIONNODE+"' field");
// Turn the configuration node into an OutputConnection
org.apache.manifoldcf.authorities.authority.AuthorityConnection authorityConnection = new org.apache.manifoldcf.authorities.authority.AuthorityConnection();
processAuthorityConnection(authorityConnection,connectionNode);
if (authorityConnection.getName() == null)
authorityConnection.setName(connectionName);
else
{
if (!authorityConnection.getName().equals(connectionName))
throw new ManifoldCFException("Connection name in path and in object must agree");
}
try
{
// Save the connection.
IAuthorityConnectionManager connectionManager = AuthorityConnectionManagerFactory.make(tc);
if (connectionManager.save(authorityConnection))
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Write mapping connection.
*/
protected static int apiWriteMappingConnection(IThreadContext tc, Configuration output, Configuration input, String connectionName)
throws ManifoldCFException
{
ConfigurationNode connectionNode = findConfigurationNode(input,API_MAPPINGCONNECTIONNODE);
if (connectionNode == null)
throw new ManifoldCFException("Input argument must have '"+API_MAPPINGCONNECTIONNODE+"' field");
// Turn the configuration node into an OutputConnection
org.apache.manifoldcf.authorities.mapping.MappingConnection mappingConnection = new org.apache.manifoldcf.authorities.mapping.MappingConnection();
processMappingConnection(mappingConnection,connectionNode);
if (mappingConnection.getName() == null)
mappingConnection.setName(connectionName);
else
{
if (!mappingConnection.getName().equals(connectionName))
throw new ManifoldCFException("Connection name in path and in object must agree");
}
try
{
// Save the connection.
IMappingConnectionManager connectionManager = MappingConnectionManagerFactory.make(tc);
if (connectionManager.save(mappingConnection))
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Write repository connection.
*/
protected static int apiWriteRepositoryConnection(IThreadContext tc, Configuration output, Configuration input, String connectionName)
throws ManifoldCFException
{
ConfigurationNode connectionNode = findConfigurationNode(input,API_REPOSITORYCONNECTIONNODE);
if (connectionNode == null)
throw new ManifoldCFException("Input argument must have '"+API_REPOSITORYCONNECTIONNODE+"' field");
// Turn the configuration node into an OutputConnection
org.apache.manifoldcf.crawler.repository.RepositoryConnection repositoryConnection = new org.apache.manifoldcf.crawler.repository.RepositoryConnection();
processRepositoryConnection(repositoryConnection,connectionNode);
if (repositoryConnection.getName() == null)
repositoryConnection.setName(connectionName);
else
{
if (!repositoryConnection.getName().equals(connectionName))
throw new ManifoldCFException("Connection name in path and in object must agree");
}
try
{
// Save the connection.
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
if (connectionManager.save(repositoryConnection))
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Reset output connection.
*/
protected static int apiWriteResetOutputConnection(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
signalOutputConnectionRedo(tc,connectionName);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return WRITERESULT_FOUND;
}
/** Execute specified write command.
*@param tc is the thread context.
*@param output is the output object, to be filled in.
*@param path is the object path.
*@param input is the input object.
*@return write result - either "not found", "found", or "created".
*/
public static int executeWriteCommand(IThreadContext tc, Configuration output, String path, Configuration input)
throws ManifoldCFException
{
if (path.startsWith("start/"))
{
Long jobID = new Long(path.substring("start/".length()));
return apiWriteStartJob(tc,output,jobID,false);
}
else if (path.startsWith("startminimal/"))
{
Long jobID = new Long(path.substring("startminimal/".length()));
return apiWriteStartJob(tc,output,jobID,true);
}
else if (path.startsWith("abort/"))
{
Long jobID = new Long(path.substring("abort/".length()));
return apiWriteAbortJob(tc,output,jobID);
}
else if (path.startsWith("restart/"))
{
Long jobID = new Long(path.substring("restart/".length()));
return apiWriteRestartJob(tc,output,jobID,false);
}
else if (path.startsWith("restartminimal/"))
{
Long jobID = new Long(path.substring("restartminimal/".length()));
return apiWriteRestartJob(tc,output,jobID,true);
}
else if (path.startsWith("pause/"))
{
Long jobID = new Long(path.substring("pause/".length()));
return apiWritePauseJob(tc,output,jobID);
}
else if (path.startsWith("resume/"))
{
Long jobID = new Long(path.substring("resume/".length()));
return apiWriteResumeJob(tc,output,jobID);
}
else if (path.startsWith("jobs/"))
{
Long jobID = new Long(path.substring("jobs/".length()));
return apiWriteJob(tc,output,input,jobID);
}
else if (path.startsWith("outputconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("outputconnections/".length()));
return apiWriteOutputConnection(tc,output,input,connectionName);
}
else if (path.startsWith("mappingconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("mappingconnections/".length()));
return apiWriteMappingConnection(tc,output,input,connectionName);
}
else if (path.startsWith("authorityconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("authorityconnections/".length()));
return apiWriteAuthorityConnection(tc,output,input,connectionName);
}
else if (path.startsWith("repositoryconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("repositoryconnections/".length()));
return apiWriteRepositoryConnection(tc,output,input,connectionName);
}
else if (path.startsWith("reset/"))
{
int firstSeparator = "reset/".length();
int secondSeparator = path.indexOf("/",firstSeparator);
if (secondSeparator == -1)
{
createErrorNode(output,"Need connection name.");
return WRITERESULT_NOTFOUND;
}
String connectionType = path.substring(firstSeparator,secondSeparator);
String connectionName = decodeAPIPathElement(path.substring(secondSeparator+1));
if (connectionType.equals("outputconnections"))
{
return apiWriteResetOutputConnection(tc,output,connectionName);
}
else
{
createErrorNode(output,"Unknown connection type '"+connectionType+"'.");
return WRITERESULT_NOTFOUND;
}
}
else
{
createErrorNode(output,"Unrecognized resource.");
return WRITERESULT_NOTFOUND;
}
}
// Delete result codes
public static final int DELETERESULT_NOTFOUND = 0;
public static final int DELETERESULT_FOUND = 1;
/** Delete a job.
*/
protected static int apiDeleteJob(IThreadContext tc, Configuration output, Long jobID)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
jobManager.deleteJob(jobID);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return DELETERESULT_FOUND;
}
/** Delete output connection.
*/
protected static int apiDeleteOutputConnection(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
connectionManager.delete(connectionName);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return DELETERESULT_FOUND;
}
/** Delete authority connection.
*/
protected static int apiDeleteAuthorityConnection(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IAuthorityConnectionManager connectionManager = AuthorityConnectionManagerFactory.make(tc);
connectionManager.delete(connectionName);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return DELETERESULT_FOUND;
}
/** Delete repository connection.
*/
protected static int apiDeleteRepositoryConnection(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
{
try
{
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
connectionManager.delete(connectionName);
}
catch (ManifoldCFException e)
{
createErrorNode(output,e);
}
return DELETERESULT_FOUND;
}
/** Execute specified delete command.
*@param tc is the thread context.
*@param output is the output object, to be filled in.
*@param path is the object path.
*@return delete result code
*/
public static int executeDeleteCommand(IThreadContext tc, Configuration output, String path)
throws ManifoldCFException
{
if (path.startsWith("jobs/"))
{
Long jobID = new Long(path.substring("jobs/".length()));
return apiDeleteJob(tc,output,jobID);
}
else if (path.startsWith("outputconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("outputconnections/".length()));
return apiDeleteOutputConnection(tc,output,connectionName);
}
else if (path.startsWith("authorityconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("authorityconnections/".length()));
return apiDeleteAuthorityConnection(tc,output,connectionName);
}
else if (path.startsWith("repositoryconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("repositoryconnections/".length()));
return apiDeleteRepositoryConnection(tc,output,connectionName);
}
else
{
createErrorNode(output,"Unrecognized resource.");
return DELETERESULT_NOTFOUND;
}
}
// The following chunk of code is responsible for formatting a job description into a set of nodes, and for reading back a formatted job description.
// This is needed to support the job-related API methods, above.
// Job node types
protected static final String JOBNODE_ID = "id";
protected static final String JOBNODE_DESCRIPTION = "description";
protected static final String JOBNODE_CONNECTIONNAME = "repository_connection";
protected static final String JOBNODE_OUTPUTNAME = "output_connection";
protected static final String JOBNODE_DOCUMENTSPECIFICATION = "document_specification";
protected static final String JOBNODE_OUTPUTSPECIFICATION = "output_specification";
protected static final String JOBNODE_STARTMODE = "start_mode";
protected static final String JOBNODE_RUNMODE = "run_mode";
protected static final String JOBNODE_HOPCOUNTMODE = "hopcount_mode";
protected static final String JOBNODE_PRIORITY = "priority";
protected static final String JOBNODE_RECRAWLINTERVAL = "recrawl_interval";
protected static final String JOBNODE_EXPIRATIONINTERVAL = "expiration_interval";
protected static final String JOBNODE_RESEEDINTERVAL = "reseed_interval";
protected static final String JOBNODE_HOPCOUNT = "hopcount";
protected static final String JOBNODE_SCHEDULE = "schedule";
protected static final String JOBNODE_LINKTYPE = "link_type";
protected static final String JOBNODE_COUNT = "count";
protected static final String JOBNODE_REQUESTMINIMUM = "requestminimum";
protected static final String JOBNODE_TIMEZONE = "timezone";
protected static final String JOBNODE_DURATION = "duration";
protected static final String JOBNODE_DAYOFWEEK = "dayofweek";
protected static final String JOBNODE_MONTHOFYEAR = "monthofyear";
protected static final String JOBNODE_DAYOFMONTH = "dayofmonth";
protected static final String JOBNODE_YEAR = "year";
protected static final String JOBNODE_HOUROFDAY = "hourofday";
protected static final String JOBNODE_MINUTESOFHOUR = "minutesofhour";
protected static final String JOBNODE_ENUMVALUE = "value";
protected static final String JOBNODE_FORCEDPARAM = "forcedparam";
protected static final String JOBNODE_PARAMNAME = "paramname";
protected static final String JOBNODE_PARAMVALUE = "paramvalue";
/** Convert a node into a job description.
*@param jobDescription is the job to be filled in.
*@param jobNode is the configuration node corresponding to the whole job itself.
*/
protected static void processJobDescription(org.apache.manifoldcf.crawler.jobs.JobDescription jobDescription, ConfigurationNode jobNode)
throws ManifoldCFException
{
// Walk through the node's children
int i = 0;
while (i < jobNode.getChildCount())
{
ConfigurationNode child = jobNode.findChild(i++);
String childType = child.getType();
if (childType.equals(JOBNODE_ID))
{
if (child.getValue() == null)
throw new ManifoldCFException("Job id node requires a value");
jobDescription.setID(new Long(child.getValue()));
}
else if (childType.equals(JOBNODE_DESCRIPTION))
{
jobDescription.setDescription(child.getValue());
}
else if (childType.equals(JOBNODE_CONNECTIONNAME))
{
jobDescription.setConnectionName(child.getValue());
}
else if (childType.equals(JOBNODE_OUTPUTNAME))
{
jobDescription.setOutputConnectionName(child.getValue());
}
else if (childType.equals(JOBNODE_DOCUMENTSPECIFICATION))
{
// Get the job's document specification, clear out the children, and copy new ones from the child.
DocumentSpecification ds = jobDescription.getSpecification();
ds.clearChildren();
int j = 0;
while (j < child.getChildCount())
{
ConfigurationNode cn = child.findChild(j++);
ds.addChild(ds.getChildCount(),new SpecificationNode(cn));
}
}
else if (childType.equals(JOBNODE_OUTPUTSPECIFICATION))
{
// Get the job's output specification, clear out the children, and copy new ones from the child.
OutputSpecification os = jobDescription.getOutputSpecification();
os.clearChildren();
int j = 0;
while (j < child.getChildCount())
{
ConfigurationNode cn = child.findChild(j++);
os.addChild(os.getChildCount(),new SpecificationNode(cn));
}
}
else if (childType.equals(JOBNODE_STARTMODE))
{
jobDescription.setStartMethod(mapToStartMode(child.getValue()));
}
else if (childType.equals(JOBNODE_RUNMODE))
{
jobDescription.setType(mapToRunMode(child.getValue()));
}
else if (childType.equals(JOBNODE_HOPCOUNTMODE))
{
jobDescription.setHopcountMode(mapToHopcountMode(child.getValue()));
}
else if (childType.equals(JOBNODE_PRIORITY))
{
try
{
jobDescription.setPriority(Integer.parseInt(child.getValue()));
}
catch (NumberFormatException e)
{
throw new ManifoldCFException(e.getMessage(),e);
}
}
else if (childType.equals(JOBNODE_RECRAWLINTERVAL))
{
jobDescription.setInterval(interpretInterval(child.getValue()));
}
else if (childType.equals(JOBNODE_EXPIRATIONINTERVAL))
{
jobDescription.setExpiration(interpretInterval(child.getValue()));
}
else if (childType.equals(JOBNODE_RESEEDINTERVAL))
{
jobDescription.setReseedInterval(interpretInterval(child.getValue()));
}
else if (childType.equals(JOBNODE_HOPCOUNT))
{
// Read the hopcount values
String linkType = null;
String hopCount = null;
int q = 0;
while (q < child.getChildCount())
{
ConfigurationNode cn = child.findChild(q++);
if (cn.getType().equals(JOBNODE_LINKTYPE))
linkType = cn.getValue();
else if (cn.getType().equals(JOBNODE_COUNT))
hopCount = cn.getValue();
else
throw new ManifoldCFException("Found an unexpected node type: '"+cn.getType()+"'");
}
if (linkType == null)
throw new ManifoldCFException("Missing required field: '"+JOBNODE_LINKTYPE+"'");
if (hopCount == null)
throw new ManifoldCFException("Missing required field: '"+JOBNODE_COUNT+"'");
jobDescription.addHopCountFilter(linkType,new Long(hopCount));
}
else if (childType.equals(JOBNODE_FORCEDPARAM))
{
// Read the forced parameter values
String paramName = null;
String paramValue = null;
for (int q = 0; q < child.getChildCount(); q++)
{
ConfigurationNode cn = child.findChild(q);
if (cn.getType().equals(JOBNODE_PARAMNAME))
paramName = cn.getValue();
else if (cn.getType().equals(JOBNODE_PARAMVALUE))
paramValue = cn.getValue();
else
throw new ManifoldCFException("Found an unexpected node type: '"+cn.getType()+"'");
}
if (paramName == null)
throw new ManifoldCFException("Missing required field: '"+JOBNODE_PARAMNAME+"'");
if (paramValue == null)
throw new ManifoldCFException("Missing required field: '"+JOBNODE_PARAMVALUE+"'");
jobDescription.addForcedMetadataValue(paramName,paramValue);
}
else if (childType.equals(JOBNODE_SCHEDULE))
{
// Create a schedule record.
String timezone = null;
Long duration = null;
boolean requestMinimum = false;
EnumeratedValues dayOfWeek = null;
EnumeratedValues monthOfYear = null;
EnumeratedValues dayOfMonth = null;
EnumeratedValues year = null;
EnumeratedValues hourOfDay = null;
EnumeratedValues minutesOfHour = null;
// Now, walk through children of the schedule node.
int q = 0;
while (q < child.getChildCount())
{
ConfigurationNode scheduleField = child.findChild(q++);
String fieldType = scheduleField.getType();
if (fieldType.equals(JOBNODE_REQUESTMINIMUM))
{
requestMinimum = scheduleField.getValue().equals("true");
}
else if (fieldType.equals(JOBNODE_TIMEZONE))
{
timezone = scheduleField.getValue();
}
else if (fieldType.equals(JOBNODE_DURATION))
{
duration = new Long(scheduleField.getValue());
}
else if (fieldType.equals(JOBNODE_DAYOFWEEK))
{
dayOfWeek = processEnumeratedValues(scheduleField);
}
else if (fieldType.equals(JOBNODE_MONTHOFYEAR))
{
monthOfYear = processEnumeratedValues(scheduleField);
}
else if (fieldType.equals(JOBNODE_YEAR))
{
year = processEnumeratedValues(scheduleField);
}
else if (fieldType.equals(JOBNODE_DAYOFMONTH))
{
dayOfMonth = processEnumeratedValues(scheduleField);
}
else if (fieldType.equals(JOBNODE_HOUROFDAY))
{
hourOfDay = processEnumeratedValues(scheduleField);
}
else if (fieldType.equals(JOBNODE_MINUTESOFHOUR))
{
minutesOfHour = processEnumeratedValues(scheduleField);
}
else
throw new ManifoldCFException("Unrecognized field in schedule record: '"+fieldType+"'");
}
ScheduleRecord sr = new ScheduleRecord(dayOfWeek,monthOfYear,dayOfMonth,year,hourOfDay,minutesOfHour,timezone,duration,requestMinimum);
// Add the schedule record to the job.
jobDescription.addScheduleRecord(sr);
}
else
throw new ManifoldCFException("Unrecognized job field: '"+childType+"'");
}
}
/** Convert a job description into a ConfigurationNode.
*@param jobNode is the node to be filled in.
*@param job is the job description.
*/
protected static void formatJobDescription(ConfigurationNode jobNode, IJobDescription job)
{
// For each field of the job, add an appropriate child node, with value.
ConfigurationNode child;
int j;
// id
if (job.getID() != null)
{
child = new ConfigurationNode(JOBNODE_ID);
child.setValue(job.getID().toString());
jobNode.addChild(jobNode.getChildCount(),child);
}
// description
if (job.getDescription() != null)
{
child = new ConfigurationNode(JOBNODE_DESCRIPTION);
child.setValue(job.getDescription());
jobNode.addChild(jobNode.getChildCount(),child);
}
// connection
if (job.getConnectionName() != null)
{
child = new ConfigurationNode(JOBNODE_CONNECTIONNAME);
child.setValue(job.getConnectionName());
jobNode.addChild(jobNode.getChildCount(),child);
}
// output connection
if (job.getOutputConnectionName() != null)
{
child = new ConfigurationNode(JOBNODE_OUTPUTNAME);
child.setValue(job.getOutputConnectionName());
jobNode.addChild(jobNode.getChildCount(),child);
}
// Document specification
DocumentSpecification ds = job.getSpecification();
child = new ConfigurationNode(JOBNODE_DOCUMENTSPECIFICATION);
j = 0;
while (j < ds.getChildCount())
{
ConfigurationNode cn = ds.getChild(j++);
child.addChild(child.getChildCount(),cn);
}
jobNode.addChild(jobNode.getChildCount(),child);
// Output specification
OutputSpecification os = job.getOutputSpecification();
child = new ConfigurationNode(JOBNODE_OUTPUTSPECIFICATION);
j = 0;
while (j < os.getChildCount())
{
ConfigurationNode cn = os.getChild(j++);
child.addChild(child.getChildCount(),cn);
}
jobNode.addChild(jobNode.getChildCount(),child);
// Start mode
child = new ConfigurationNode(JOBNODE_STARTMODE);
child.setValue(startModeMap(job.getStartMethod()));
jobNode.addChild(jobNode.getChildCount(),child);
// Run mode
child = new ConfigurationNode(JOBNODE_RUNMODE);
child.setValue(runModeMap(job.getType()));
jobNode.addChild(jobNode.getChildCount(),child);
// Hopcount mode
child = new ConfigurationNode(JOBNODE_HOPCOUNTMODE);
child.setValue(hopcountModeMap(job.getHopcountMode()));
jobNode.addChild(jobNode.getChildCount(),child);
// Priority
child = new ConfigurationNode(JOBNODE_PRIORITY);
child.setValue(Integer.toString(job.getPriority()));
jobNode.addChild(jobNode.getChildCount(),child);
// Recrawl interval
child = new ConfigurationNode(JOBNODE_RECRAWLINTERVAL);
child.setValue((job.getInterval()==null)?"infinite":job.getInterval().toString());
jobNode.addChild(jobNode.getChildCount(),child);
child = new ConfigurationNode(JOBNODE_EXPIRATIONINTERVAL);
child.setValue((job.getExpiration()==null)?"infinite":job.getExpiration().toString());
jobNode.addChild(jobNode.getChildCount(),child);
child = new ConfigurationNode(JOBNODE_RESEEDINTERVAL);
child.setValue((job.getReseedInterval()==null)?"infinite":job.getReseedInterval().toString());
jobNode.addChild(jobNode.getChildCount(),child);
// Hopcount records
Map filters = job.getHopCountFilters();
Iterator iter = filters.keySet().iterator();
while (iter.hasNext())
{
String linkType = (String)iter.next();
Long hopCount = (Long)filters.get(linkType);
child = new ConfigurationNode(JOBNODE_HOPCOUNT);
ConfigurationNode cn;
cn = new ConfigurationNode(JOBNODE_LINKTYPE);
cn.setValue(linkType);
child.addChild(child.getChildCount(),cn);
cn = new ConfigurationNode(JOBNODE_COUNT);
cn.setValue(hopCount.toString());
child.addChild(child.getChildCount(),cn);
jobNode.addChild(jobNode.getChildCount(),child);
}
// Forced metadata records
Map<String,Set<String>> forcedMetadata = job.getForcedMetadata();
for (String paramName : forcedMetadata.keySet())
{
Set<String> values = forcedMetadata.get(paramName);
for (String paramValue : values)
{
child = new ConfigurationNode(JOBNODE_FORCEDPARAM);
ConfigurationNode cn;
cn = new ConfigurationNode(JOBNODE_PARAMNAME);
cn.setValue(paramName);
child.addChild(child.getChildCount(),cn);
cn = new ConfigurationNode(JOBNODE_PARAMVALUE);
cn.setValue(paramValue);
child.addChild(child.getChildCount(),cn);
jobNode.addChild(jobNode.getChildCount(),child);
}
}
// Schedule records
j = 0;
while (j < job.getScheduleRecordCount())
{
ScheduleRecord sr = job.getScheduleRecord(j++);
child = new ConfigurationNode(JOBNODE_SCHEDULE);
ConfigurationNode recordChild;
// requestminimum
recordChild = new ConfigurationNode(JOBNODE_REQUESTMINIMUM);
recordChild.setValue(sr.getRequestMinimum()?"true":"false");
child.addChild(child.getChildCount(),recordChild);
// timezone
if (sr.getTimezone() != null)
{
recordChild = new ConfigurationNode(JOBNODE_TIMEZONE);
recordChild.setValue(sr.getTimezone());
child.addChild(child.getChildCount(),recordChild);
}
// duration
if (sr.getDuration() != null)
{
recordChild = new ConfigurationNode(JOBNODE_DURATION);
recordChild.setValue(sr.getDuration().toString());
child.addChild(child.getChildCount(),recordChild);
}
// Schedule specification values
// day of week
if (sr.getDayOfWeek() != null)
formatEnumeratedValues(child,JOBNODE_DAYOFWEEK,sr.getDayOfWeek());
if (sr.getMonthOfYear() != null)
formatEnumeratedValues(child,JOBNODE_MONTHOFYEAR,sr.getMonthOfYear());
if (sr.getDayOfMonth() != null)
formatEnumeratedValues(child,JOBNODE_DAYOFMONTH,sr.getDayOfMonth());
if (sr.getYear() != null)
formatEnumeratedValues(child,JOBNODE_YEAR,sr.getYear());
if (sr.getHourOfDay() != null)
formatEnumeratedValues(child,JOBNODE_HOUROFDAY,sr.getHourOfDay());
if (sr.getMinutesOfHour() != null)
formatEnumeratedValues(child,JOBNODE_MINUTESOFHOUR,sr.getMinutesOfHour());
jobNode.addChild(jobNode.getChildCount(),child);
}
}
protected static void formatEnumeratedValues(ConfigurationNode recordNode, String childType, EnumeratedValues value)
{
ConfigurationNode child = new ConfigurationNode(childType);
Iterator iter = value.getValues();
while (iter.hasNext())
{
Integer theValue = (Integer)iter.next();
ConfigurationNode valueNode = new ConfigurationNode(JOBNODE_ENUMVALUE);
valueNode.setValue(theValue.toString());
child.addChild(child.getChildCount(),valueNode);
}
recordNode.addChild(recordNode.getChildCount(),child);
}
protected static EnumeratedValues processEnumeratedValues(ConfigurationNode fieldNode)
throws ManifoldCFException
{
ArrayList values = new ArrayList();
int i = 0;
while (i < fieldNode.getChildCount())
{
ConfigurationNode cn = fieldNode.findChild(i++);
if (cn.getType().equals(JOBNODE_ENUMVALUE))
{
try
{
values.add(new Integer(cn.getValue()));
}
catch (NumberFormatException e)
{
throw new ManifoldCFException("Error processing enumerated value node: "+e.getMessage(),e);
}
}
else
throw new ManifoldCFException("Error processing enumerated value nodes: Unrecognized node type '"+cn.getType()+"'");
}
return new EnumeratedValues(values);
}
protected static String presentInterval(Long interval)
{
if (interval == null)
return "infinite";
return interval.toString();
}
protected static Long interpretInterval(String interval)
throws ManifoldCFException
{
if (interval == null || interval.equals("infinite"))
return null;
else
return new Long(interval);
}
protected static String startModeMap(int startMethod)
{
switch (startMethod)
{
case IJobDescription.START_WINDOWBEGIN:
return "schedule window start";
case IJobDescription.START_WINDOWINSIDE:
return "schedule window anytime";
case IJobDescription.START_DISABLE:
return "manual";
default:
return "unknown";
}
}
protected static int mapToStartMode(String startMethod)
throws ManifoldCFException
{
if (startMethod.equals("schedule window start"))
return IJobDescription.START_WINDOWBEGIN;
else if (startMethod.equals("schedule window anytime"))
return IJobDescription.START_WINDOWINSIDE;
else if (startMethod.equals("manual"))
return IJobDescription.START_DISABLE;
else
throw new ManifoldCFException("Unrecognized start method: '"+startMethod+"'");
}
protected static String runModeMap(int type)
{
switch (type)
{
case IJobDescription.TYPE_CONTINUOUS:
return "continuous";
case IJobDescription.TYPE_SPECIFIED:
return "scan once";
default:
return "unknown";
}
}
protected static int mapToRunMode(String mode)
throws ManifoldCFException
{
if (mode.equals("continuous"))
return IJobDescription.TYPE_CONTINUOUS;
else if (mode.equals("scan once"))
return IJobDescription.TYPE_SPECIFIED;
else
throw new ManifoldCFException("Unrecognized run method: '"+mode+"'");
}
protected static String hopcountModeMap(int mode)
{
switch (mode)
{
case IJobDescription.HOPCOUNT_ACCURATE:
return "accurate";
case IJobDescription.HOPCOUNT_NODELETE:
return "no delete";
case IJobDescription.HOPCOUNT_NEVERDELETE:
return "never delete";
default:
return "unknown";
}
}
protected static int mapToHopcountMode(String mode)
throws ManifoldCFException
{
if (mode.equals("accurate"))
return IJobDescription.HOPCOUNT_ACCURATE;
else if (mode.equals("no delete"))
return IJobDescription.HOPCOUNT_NODELETE;
else if (mode.equals("never delete"))
return IJobDescription.HOPCOUNT_NEVERDELETE;
else
throw new ManifoldCFException("Unrecognized hopcount method: '"+mode+"'");
}
// End of job API support code.
// The following chunk of code supports job statuses in the API. Only a formatting method is required, since we never "save" a status.
// Node types used to handle job statuses.
protected static final String JOBSTATUSNODE_JOBID = "job_id";
protected static final String JOBSTATUSNODE_STATUS = "status";
protected static final String JOBSTATUSNODE_ERRORTEXT = "errortext";
protected static final String JOBSTATUSNODE_STARTTIME = "start_time";
protected static final String JOBSTATUSNODE_ENDTIME = "end_time";
protected static final String JOBSTATUSNODE_DOCUMENTSINQUEUE = "documents_in_queue";
protected static final String JOBSTATUSNODE_DOCUMENTSOUTSTANDING = "documents_outstanding";
protected static final String JOBSTATUSNODE_DOCUMENTSPROCESSED = "documents_processed";
/** Format a job status.
*/
protected static void formatJobStatus(ConfigurationNode jobStatusNode, JobStatus jobStatus)
{
// For each field of the job, add an appropriate child node, with value.
ConfigurationNode child;
int j;
// id
child = new ConfigurationNode(JOBSTATUSNODE_JOBID);
child.setValue(jobStatus.getJobID().toString());
jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
// status
child = new ConfigurationNode(JOBSTATUSNODE_STATUS);
child.setValue(statusMap(jobStatus.getStatus()));
jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
// error text
if (jobStatus.getErrorText() != null)
{
child = new ConfigurationNode(JOBSTATUSNODE_ERRORTEXT);
child.setValue(jobStatus.getErrorText());
jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
}
// start time
if (jobStatus.getStartTime() != -1L)
{
child = new ConfigurationNode(JOBSTATUSNODE_STARTTIME);
child.setValue(new Long(jobStatus.getStartTime()).toString());
jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
}
// end time
if (jobStatus.getEndTime() != -1L)
{
child = new ConfigurationNode(JOBSTATUSNODE_ENDTIME);
child.setValue(new Long(jobStatus.getEndTime()).toString());
jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
}
// documents in queue
child = new ConfigurationNode(JOBSTATUSNODE_DOCUMENTSINQUEUE);
child.setValue(new Long(jobStatus.getDocumentsInQueue()).toString());
jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
// documents outstanding
child = new ConfigurationNode(JOBSTATUSNODE_DOCUMENTSOUTSTANDING);
child.setValue(new Long(jobStatus.getDocumentsOutstanding()).toString());
jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
// documents processed
child = new ConfigurationNode(JOBSTATUSNODE_DOCUMENTSPROCESSED);
child.setValue(new Long(jobStatus.getDocumentsProcessed()).toString());
jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
}
protected static String statusMap(int status)
{
switch (status)
{
case JobStatus.JOBSTATUS_NOTYETRUN:
return "not yet run";
case JobStatus.JOBSTATUS_RUNNING:
return "running";
case JobStatus.JOBSTATUS_STOPPING:
return "stopping";
case JobStatus.JOBSTATUS_RESUMING:
return "resuming";
case JobStatus.JOBSTATUS_PAUSED:
return "paused";
case JobStatus.JOBSTATUS_COMPLETED:
return "done";
case JobStatus.JOBSTATUS_WINDOWWAIT:
return "waiting";
case JobStatus.JOBSTATUS_STARTING:
return "starting up";
case JobStatus.JOBSTATUS_DESTRUCTING:
return "cleaning up";
case JobStatus.JOBSTATUS_ERROR:
return "error";
case JobStatus.JOBSTATUS_ABORTING:
return "aborting";
case JobStatus.JOBSTATUS_RESTARTING:
return "restarting";
case JobStatus.JOBSTATUS_RUNNING_UNINSTALLED:
return "running no connector";
case JobStatus.JOBSTATUS_JOBENDCLEANUP:
return "terminating";
case JobStatus.JOBSTATUS_JOBENDNOTIFICATION:
return "notifying";
default:
return "unknown";
}
}
// End of jobstatus API support.
// Connection API
protected static final String CONNECTIONNODE_ISNEW = "isnew";
protected static final String CONNECTIONNODE_NAME = "name";
protected static final String CONNECTIONNODE_CLASSNAME = "class_name";
protected static final String CONNECTIONNODE_MAXCONNECTIONS = "max_connections";
protected static final String CONNECTIONNODE_DESCRIPTION = "description";
protected static final String CONNECTIONNODE_PREREQUISITE = "prerequisite";
protected static final String CONNECTIONNODE_CONFIGURATION = "configuration";
protected static final String CONNECTIONNODE_ACLAUTHORITY = "acl_authority";
protected static final String CONNECTIONNODE_THROTTLE = "throttle";
protected static final String CONNECTIONNODE_MATCH = "match";
protected static final String CONNECTIONNODE_MATCHDESCRIPTION = "match_description";
protected static final String CONNECTIONNODE_RATE = "rate";
// Output connection API support.
/** Convert input hierarchy into an OutputConnection object.
*/
protected static void processOutputConnection(org.apache.manifoldcf.agents.outputconnection.OutputConnection connection, ConfigurationNode connectionNode)
throws ManifoldCFException
{
// Walk through the node's children
int i = 0;
while (i < connectionNode.getChildCount())
{
ConfigurationNode child = connectionNode.findChild(i++);
String childType = child.getType();
if (childType.equals(CONNECTIONNODE_ISNEW))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection isnew node requires a value");
connection.setIsNew(child.getValue().equals("true"));
}
else if (childType.equals(CONNECTIONNODE_NAME))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection name node requires a value");
connection.setName(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_CLASSNAME))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection classname node requires a value");
connection.setClassName(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_MAXCONNECTIONS))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection maxconnections node requires a value");
try
{
connection.setMaxConnections(Integer.parseInt(child.getValue()));
}
catch (NumberFormatException e)
{
throw new ManifoldCFException("Error parsing max connections: "+e.getMessage(),e);
}
}
else if (childType.equals(CONNECTIONNODE_DESCRIPTION))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection description node requires a value");
connection.setDescription(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_CONFIGURATION))
{
// Get the connection's configuration, clear out the children, and copy new ones from the child.
ConfigParams cp = connection.getConfigParams();
cp.clearChildren();
int j = 0;
while (j < child.getChildCount())
{
ConfigurationNode cn = child.findChild(j++);
cp.addChild(cp.getChildCount(),new ConfigNode(cn));
}
}
else
throw new ManifoldCFException("Unrecognized output connection field: '"+childType+"'");
}
if (connection.getClassName() == null)
throw new ManifoldCFException("Missing connection field: '"+CONNECTIONNODE_CLASSNAME+"'");
}
/** Format an output connection.
*/
protected static void formatOutputConnection(ConfigurationNode connectionNode, IOutputConnection connection)
{
ConfigurationNode child;
int j;
child = new ConfigurationNode(CONNECTIONNODE_ISNEW);
child.setValue(connection.getIsNew()?"true":"false");
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_NAME);
child.setValue(connection.getName());
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_CLASSNAME);
child.setValue(connection.getClassName());
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_MAXCONNECTIONS);
child.setValue(Integer.toString(connection.getMaxConnections()));
connectionNode.addChild(connectionNode.getChildCount(),child);
if (connection.getDescription() != null)
{
child = new ConfigurationNode(CONNECTIONNODE_DESCRIPTION);
child.setValue(connection.getDescription());
connectionNode.addChild(connectionNode.getChildCount(),child);
}
ConfigParams cp = connection.getConfigParams();
child = new ConfigurationNode(CONNECTIONNODE_CONFIGURATION);
j = 0;
while (j < cp.getChildCount())
{
ConfigurationNode cn = cp.findChild(j++);
child.addChild(child.getChildCount(),cn);
}
connectionNode.addChild(connectionNode.getChildCount(),child);
}
// Authority connection API support
/** Convert input hierarchy into an AuthorityConnection object.
*/
protected static void processAuthorityConnection(org.apache.manifoldcf.authorities.authority.AuthorityConnection connection, ConfigurationNode connectionNode)
throws ManifoldCFException
{
// Walk through the node's children
int i = 0;
while (i < connectionNode.getChildCount())
{
ConfigurationNode child = connectionNode.findChild(i++);
String childType = child.getType();
if (childType.equals(CONNECTIONNODE_ISNEW))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection isnew node requires a value");
connection.setIsNew(child.getValue().equals("true"));
}
else if (childType.equals(CONNECTIONNODE_NAME))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection name node requires a value");
connection.setName(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_CLASSNAME))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection classname node requires a value");
connection.setClassName(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_MAXCONNECTIONS))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection maxconnections node requires a value");
try
{
connection.setMaxConnections(Integer.parseInt(child.getValue()));
}
catch (NumberFormatException e)
{
throw new ManifoldCFException("Error parsing max connections: "+e.getMessage(),e);
}
}
else if (childType.equals(CONNECTIONNODE_PREREQUISITE))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection prerequisite node requires a value");
connection.setPrerequisiteMapping(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_DESCRIPTION))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection description node requires a value");
connection.setDescription(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_CONFIGURATION))
{
// Get the connection's configuration, clear out the children, and copy new ones from the child.
ConfigParams cp = connection.getConfigParams();
cp.clearChildren();
int j = 0;
while (j < child.getChildCount())
{
ConfigurationNode cn = child.findChild(j++);
cp.addChild(cp.getChildCount(),new ConfigNode(cn));
}
}
else
throw new ManifoldCFException("Unrecognized authority connection field: '"+childType+"'");
}
if (connection.getClassName() == null)
throw new ManifoldCFException("Missing connection field: '"+CONNECTIONNODE_CLASSNAME+"'");
}
/** Format an authority connection.
*/
protected static void formatAuthorityConnection(ConfigurationNode connectionNode, IAuthorityConnection connection)
{
ConfigurationNode child;
int j;
child = new ConfigurationNode(CONNECTIONNODE_ISNEW);
child.setValue(connection.getIsNew()?"true":"false");
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_NAME);
child.setValue(connection.getName());
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_CLASSNAME);
child.setValue(connection.getClassName());
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_MAXCONNECTIONS);
child.setValue(Integer.toString(connection.getMaxConnections()));
connectionNode.addChild(connectionNode.getChildCount(),child);
if (connection.getPrerequisiteMapping() != null)
{
child = new ConfigurationNode(CONNECTIONNODE_PREREQUISITE);
child.setValue(connection.getPrerequisiteMapping());
connectionNode.addChild(connectionNode.getChildCount(),child);
}
if (connection.getDescription() != null)
{
child = new ConfigurationNode(CONNECTIONNODE_DESCRIPTION);
child.setValue(connection.getDescription());
connectionNode.addChild(connectionNode.getChildCount(),child);
}
ConfigParams cp = connection.getConfigParams();
child = new ConfigurationNode(CONNECTIONNODE_CONFIGURATION);
j = 0;
while (j < cp.getChildCount())
{
ConfigurationNode cn = cp.findChild(j++);
child.addChild(child.getChildCount(),cn);
}
connectionNode.addChild(connectionNode.getChildCount(),child);
}
// Mapping connection API methods
/** Convert input hierarchy into an MappingConnection object.
*/
protected static void processMappingConnection(org.apache.manifoldcf.authorities.mapping.MappingConnection connection, ConfigurationNode connectionNode)
throws ManifoldCFException
{
// Walk through the node's children
int i = 0;
while (i < connectionNode.getChildCount())
{
ConfigurationNode child = connectionNode.findChild(i++);
String childType = child.getType();
if (childType.equals(CONNECTIONNODE_ISNEW))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection isnew node requires a value");
connection.setIsNew(child.getValue().equals("true"));
}
else if (childType.equals(CONNECTIONNODE_NAME))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection name node requires a value");
connection.setName(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_CLASSNAME))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection classname node requires a value");
connection.setClassName(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_MAXCONNECTIONS))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection maxconnections node requires a value");
try
{
connection.setMaxConnections(Integer.parseInt(child.getValue()));
}
catch (NumberFormatException e)
{
throw new ManifoldCFException("Error parsing max connections: "+e.getMessage(),e);
}
}
else if (childType.equals(CONNECTIONNODE_PREREQUISITE))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection prerequisite node requires a value");
connection.setPrerequisiteMapping(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_DESCRIPTION))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection description node requires a value");
connection.setDescription(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_CONFIGURATION))
{
// Get the connection's configuration, clear out the children, and copy new ones from the child.
ConfigParams cp = connection.getConfigParams();
cp.clearChildren();
int j = 0;
while (j < child.getChildCount())
{
ConfigurationNode cn = child.findChild(j++);
cp.addChild(cp.getChildCount(),new ConfigNode(cn));
}
}
else
throw new ManifoldCFException("Unrecognized mapping connection field: '"+childType+"'");
}
if (connection.getClassName() == null)
throw new ManifoldCFException("Missing connection field: '"+CONNECTIONNODE_CLASSNAME+"'");
}
/** Format a mapping connection.
*/
protected static void formatMappingConnection(ConfigurationNode connectionNode, IMappingConnection connection)
{
ConfigurationNode child;
int j;
child = new ConfigurationNode(CONNECTIONNODE_ISNEW);
child.setValue(connection.getIsNew()?"true":"false");
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_NAME);
child.setValue(connection.getName());
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_CLASSNAME);
child.setValue(connection.getClassName());
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_MAXCONNECTIONS);
child.setValue(Integer.toString(connection.getMaxConnections()));
connectionNode.addChild(connectionNode.getChildCount(),child);
if (connection.getPrerequisiteMapping() != null)
{
child = new ConfigurationNode(CONNECTIONNODE_PREREQUISITE);
child.setValue(connection.getPrerequisiteMapping());
connectionNode.addChild(connectionNode.getChildCount(),child);
}
if (connection.getDescription() != null)
{
child = new ConfigurationNode(CONNECTIONNODE_DESCRIPTION);
child.setValue(connection.getDescription());
connectionNode.addChild(connectionNode.getChildCount(),child);
}
ConfigParams cp = connection.getConfigParams();
child = new ConfigurationNode(CONNECTIONNODE_CONFIGURATION);
j = 0;
while (j < cp.getChildCount())
{
ConfigurationNode cn = cp.findChild(j++);
child.addChild(child.getChildCount(),cn);
}
connectionNode.addChild(connectionNode.getChildCount(),child);
}
// Repository connection API support methods
/** Convert input hierarchy into a RepositoryConnection object.
*/
protected static void processRepositoryConnection(org.apache.manifoldcf.crawler.repository.RepositoryConnection connection, ConfigurationNode connectionNode)
throws ManifoldCFException
{
// Walk through the node's children
int i = 0;
while (i < connectionNode.getChildCount())
{
ConfigurationNode child = connectionNode.findChild(i++);
String childType = child.getType();
if (childType.equals(CONNECTIONNODE_ISNEW))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection isnew node requires a value");
connection.setIsNew(child.getValue().equals("true"));
}
else if (childType.equals(CONNECTIONNODE_NAME))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection name node requires a value");
connection.setName(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_CLASSNAME))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection classname node requires a value");
connection.setClassName(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_MAXCONNECTIONS))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection maxconnections node requires a value");
try
{
connection.setMaxConnections(Integer.parseInt(child.getValue()));
}
catch (NumberFormatException e)
{
throw new ManifoldCFException("Error parsing max connections: "+e.getMessage(),e);
}
}
else if (childType.equals(CONNECTIONNODE_DESCRIPTION))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection description node requires a value");
connection.setDescription(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_CONFIGURATION))
{
// Get the connection's configuration, clear out the children, and copy new ones from the child.
ConfigParams cp = connection.getConfigParams();
cp.clearChildren();
int j = 0;
while (j < child.getChildCount())
{
ConfigurationNode cn = child.findChild(j++);
cp.addChild(cp.getChildCount(),new ConfigNode(cn));
}
}
else if (childType.equals(CONNECTIONNODE_ACLAUTHORITY))
{
if (child.getValue() == null)
throw new ManifoldCFException("Connection aclauthority node requires a value");
connection.setACLAuthority(child.getValue());
}
else if (childType.equals(CONNECTIONNODE_THROTTLE))
{
String match = null;
String description = null;
Float rate = null;
int q = 0;
while (q < child.getChildCount())
{
ConfigurationNode throttleField = child.findChild(q++);
String fieldType = throttleField.getType();
if (fieldType.equals(CONNECTIONNODE_MATCH))
{
match = throttleField.getValue();
}
else if (fieldType.equals(CONNECTIONNODE_MATCHDESCRIPTION))
{
description = throttleField.getValue();
}
else if (fieldType.equals(CONNECTIONNODE_RATE))
{
rate = new Float(throttleField.getValue());
}
else
throw new ManifoldCFException("Unrecognized throttle field: '"+fieldType+"'");
}
if (match == null)
throw new ManifoldCFException("Missing throttle field: '"+CONNECTIONNODE_MATCH+"'");
if (rate == null)
throw new ManifoldCFException("Missing throttle field: '"+CONNECTIONNODE_RATE+"'");
connection.addThrottleValue(match,description,rate.floatValue());
}
else
throw new ManifoldCFException("Unrecognized repository connection field: '"+childType+"'");
}
if (connection.getClassName() == null)
throw new ManifoldCFException("Missing connection field: '"+CONNECTIONNODE_CLASSNAME+"'");
}
/** Format a repository connection.
*/
protected static void formatRepositoryConnection(ConfigurationNode connectionNode, IRepositoryConnection connection)
{
ConfigurationNode child;
int j;
child = new ConfigurationNode(CONNECTIONNODE_ISNEW);
child.setValue(connection.getIsNew()?"true":"false");
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_NAME);
child.setValue(connection.getName());
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_CLASSNAME);
child.setValue(connection.getClassName());
connectionNode.addChild(connectionNode.getChildCount(),child);
child = new ConfigurationNode(CONNECTIONNODE_MAXCONNECTIONS);
child.setValue(Integer.toString(connection.getMaxConnections()));
connectionNode.addChild(connectionNode.getChildCount(),child);
if (connection.getDescription() != null)
{
child = new ConfigurationNode(CONNECTIONNODE_DESCRIPTION);
child.setValue(connection.getDescription());
connectionNode.addChild(connectionNode.getChildCount(),child);
}
ConfigParams cp = connection.getConfigParams();
child = new ConfigurationNode(CONNECTIONNODE_CONFIGURATION);
j = 0;
while (j < cp.getChildCount())
{
ConfigurationNode cn = cp.findChild(j++);
child.addChild(child.getChildCount(),cn);
}
connectionNode.addChild(connectionNode.getChildCount(),child);
if (connection.getACLAuthority() != null)
{
child = new ConfigurationNode(CONNECTIONNODE_ACLAUTHORITY);
child.setValue(connection.getACLAuthority());
connectionNode.addChild(connectionNode.getChildCount(),child);
}
String[] throttles = connection.getThrottles();
j = 0;
while (j < throttles.length)
{
String match = throttles[j++];
String description = connection.getThrottleDescription(match);
float rate = connection.getThrottleValue(match);
child = new ConfigurationNode(CONNECTIONNODE_THROTTLE);
ConfigurationNode throttleChildNode;
throttleChildNode = new ConfigurationNode(CONNECTIONNODE_MATCH);
throttleChildNode.setValue(match);
child.addChild(child.getChildCount(),throttleChildNode);
if (description != null)
{
throttleChildNode = new ConfigurationNode(CONNECTIONNODE_MATCHDESCRIPTION);
throttleChildNode.setValue(description);
child.addChild(child.getChildCount(),throttleChildNode);
}
throttleChildNode = new ConfigurationNode(CONNECTIONNODE_RATE);
throttleChildNode.setValue(new Float(rate).toString());
child.addChild(child.getChildCount(),throttleChildNode);
connectionNode.addChild(connectionNode.getChildCount(),child);
}
}
// End of connection API code
}