/*
* Copyright 2004-2014 SmartBear Software
*
* Licensed under the EUPL, Version 1.1 or - as soon as they will be approved by the European Commission - subsequent
* versions of the EUPL (the "Licence");
* You may not use this work except in compliance with the Licence.
* You may obtain a copy of the Licence at:
*
* http://ec.europa.eu/idabc/eupl
*
* Unless required by applicable law or agreed to in writing, software distributed under the Licence is
* distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the Licence for the specific language governing permissions and limitations
* under the Licence.
*/
package com.eviware.soapui.tools;
import java.io.File;
import java.io.FileOutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.cli.CommandLine;
import com.eviware.soapui.SoapUI;
import com.eviware.soapui.impl.wsdl.WsdlProject;
import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
import com.eviware.soapui.impl.wsdl.testcase.WsdlProjectRunner;
import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCaseRunner;
import com.eviware.soapui.impl.wsdl.testcase.WsdlTestSuiteRunner;
import com.eviware.soapui.impl.wsdl.teststeps.WsdlRunTestCaseTestStep;
import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
import com.eviware.soapui.model.iface.Attachment;
import com.eviware.soapui.model.iface.MessageExchange;
import com.eviware.soapui.model.project.ProjectFactoryRegistry;
import com.eviware.soapui.model.support.ModelSupport;
import com.eviware.soapui.model.support.ProjectRunListenerAdapter;
import com.eviware.soapui.model.testsuite.Assertable;
import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
import com.eviware.soapui.model.testsuite.AssertionError;
import com.eviware.soapui.model.testsuite.ProjectRunContext;
import com.eviware.soapui.model.testsuite.ProjectRunner;
import com.eviware.soapui.model.testsuite.TestAssertion;
import com.eviware.soapui.model.testsuite.TestCase;
import com.eviware.soapui.model.testsuite.TestCaseRunContext;
import com.eviware.soapui.model.testsuite.TestCaseRunner;
import com.eviware.soapui.model.testsuite.TestRunner.Status;
import com.eviware.soapui.model.testsuite.TestStep;
import com.eviware.soapui.model.testsuite.TestStepResult;
import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
import com.eviware.soapui.model.testsuite.TestSuite;
import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
import com.eviware.soapui.model.testsuite.TestSuiteRunner;
import com.eviware.soapui.report.JUnitReportCollector;
import com.eviware.soapui.report.JUnitSecurityReportCollector;
import com.eviware.soapui.report.TestCaseRunLogReport;
import com.eviware.soapui.support.StringUtils;
import com.eviware.soapui.support.Tools;
import com.eviware.soapui.support.types.StringToObjectMap;
/**
* Standalone test-runner used from maven-plugin, can also be used from
* command-line (see xdocs) or directly from other classes.
* <p>
* For standalone usage, set the project file (with setProjectFile) and other
* desired properties before calling run
* </p>
*
* @author Ole.Matzura
*/
public class SoapUITestCaseRunner extends AbstractSoapUITestRunner {
public static final String SOAPUI_EXPORT_SEPARATOR = "soapui.export.separator";
public static final String TITLE = "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
private String testSuite;
private String testCase;
private List<TestAssertion> assertions = new ArrayList<TestAssertion>();
private Map<TestAssertion, WsdlTestStepResult> assertionResults = new HashMap<TestAssertion, WsdlTestStepResult>();
private List<TestCase> failedTests = new ArrayList<TestCase>();
private int testSuiteCount;
private int testCaseCount;
private int testStepCount;
private int testAssertionCount;
private boolean printReport;
private boolean printAlertSiteReport;
private boolean exportAll;
private boolean ignoreErrors;
private boolean junitReport;
private int exportCount;
private int maxErrors = 5;
private JUnitReportCollector reportCollector;
private String projectPassword;
private boolean saveAfterRun;
private TestCaseRunLogReport testCaseRunLogReport;
/**
* Runs the tests in the specified soapUI project file, see SoapUI xdocs for
* details.
*
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
System.exit(new SoapUITestCaseRunner().runFromCommandLine(args));
}
@Override
protected boolean processCommandLine(CommandLine cmd) {
String message = "";
if (cmd.hasOption("e")) {
setEndpoint(cmd.getOptionValue("e"));
}
if (cmd.hasOption("s")) {
String testSuite = getCommandLineOptionSubstSpace(cmd, "s");
setTestSuite(testSuite);
}
if (cmd.hasOption("c")) {
String testCase = getCommandLineOptionSubstSpace(cmd, "c");
setTestCase(testCase);
}
if (cmd.hasOption("u")) {
setUsername(cmd.getOptionValue("u"));
}
if (cmd.hasOption("p")) {
setPassword(cmd.getOptionValue("p"));
}
if (cmd.hasOption("w")) {
setWssPasswordType(cmd.getOptionValue("w"));
}
if (cmd.hasOption("d")) {
setDomain(cmd.getOptionValue("d"));
}
if (cmd.hasOption("h")) {
setHost(cmd.getOptionValue("h"));
}
if (cmd.hasOption("f")) {
setOutputFolder(getCommandLineOptionSubstSpace(cmd, "f"));
}
if (cmd.hasOption("t")) {
setSettingsFile(getCommandLineOptionSubstSpace(cmd, "t"));
}
if (cmd.hasOption("x")) {
setProjectPassword(cmd.getOptionValue("x"));
}
if (cmd.hasOption("v")) {
setSoapUISettingsPassword(cmd.getOptionValue("v"));
}
if (cmd.hasOption("D")) {
setSystemProperties(cmd.getOptionValues("D"));
}
if( cmd.hasOption( "H" ) )
{
setCustomHeaders( cmd.getOptionValues( "H" ) );
}
if (cmd.hasOption("G")) {
setGlobalProperties(cmd.getOptionValues("G"));
}
if (cmd.hasOption("P")) {
setProjectProperties(cmd.getOptionValues("P"));
}
setIgnoreError(cmd.hasOption("I"));
setEnableUI(cmd.hasOption("i"));
setPrintReport(cmd.hasOption("r"));
setPrintAlertSiteReport(cmd.hasOption("M"));
setExportAll(cmd.hasOption("a"));
if (cmd.hasOption("A")) {
setExportAll(true);
System.setProperty(SOAPUI_EXPORT_SEPARATOR, File.separator);
}
setJUnitReport(cmd.hasOption("j"));
if (cmd.hasOption("m")) {
setMaxErrors(Integer.parseInt(cmd.getOptionValue("m")));
}
setSaveAfterRun(cmd.hasOption("S"));
if (message.length() > 0) {
log.error(message);
return false;
}
return true;
}
public void setMaxErrors(int maxErrors) {
this.maxErrors = maxErrors;
}
protected int getMaxErrors() {
return maxErrors;
}
public void setSaveAfterRun(boolean saveAfterRun) {
this.saveAfterRun = saveAfterRun;
}
@Override
public void setProjectPassword(String projectPassword) {
this.projectPassword = projectPassword;
}
@Override
public String getProjectPassword() {
return projectPassword;
}
@Override
protected SoapUIOptions initCommandLineOptions()
{
SoapUIOptions options = new SoapUIOptions( "testrunner" );
options.addOption( "e", true, "Sets the endpoint" );
options.addOption( "s", true, "Sets the testsuite" );
options.addOption( "c", true, "Sets the testcase" );
options.addOption( "u", true, "Sets the username" );
options.addOption( "p", true, "Sets the password" );
options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
options.addOption( "i", false, "Enables Swing UI for scripts" );
options.addOption( "d", true, "Sets the domain" );
options.addOption( "h", true, "Sets the host" );
options.addOption( "r", false, "Prints a small summary report" );
options.addOption( "M", false, "Creates a Test Run Log Report in XML format" );
options.addOption( "f", true, "Sets the output folder to export results to" );
options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
options.addOption( "m", false, "Sets the maximum number of TestStep errors to save for each testcase" );
options.addOption( "a", false, "Turns on exporting of all results" );
options.addOption( "A", false, "Turns on exporting of all results using folders instead of long filenames" );
options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
options.addOption( "D", true, "Sets system property with name=value" );
options.addOption( "G", true, "Sets global property with name=value" );
options.addOption( "P", true, "Sets or overrides project property with name=value" );
options.addOption( "I", false, "Do not stop if error occurs, ignore them" );
options.addOption( "S", false , "Saves the project after running the tests" );
options.addOption( "H", true , "Adds a custom HTTP Header to all outgoing requests (name=value), can be specified multiple times" );
return options;
}
/**
* Add console appender to groovy log
*/
public void setExportAll(boolean exportAll) {
this.exportAll = exportAll;
}
public void setJUnitReport(boolean junitReport) {
this.junitReport = junitReport;
if (junitReport) {
reportCollector = createJUnitSecurityReportCollector();
}
}
protected JUnitSecurityReportCollector createJUnitSecurityReportCollector() {
return new JUnitSecurityReportCollector();
}
public SoapUITestCaseRunner() {
super(SoapUITestCaseRunner.TITLE);
}
public SoapUITestCaseRunner(String title) {
super(title);
}
/**
* Controls if a short test summary should be printed after the test runs
*
* @param printReport a flag controlling if a summary should be printed
*/
public void setPrintReport(boolean printReport) {
this.printReport = printReport;
}
public void setPrintAlertSiteReport(boolean printAlertSiteReport) {
this.printAlertSiteReport = printAlertSiteReport;
}
public boolean isPrintAlertSiteReport() {
return printAlertSiteReport;
}
public void setIgnoreError(boolean ignoreErrors) {
this.ignoreErrors = ignoreErrors;
}
@Override
public boolean runRunner() throws Exception {
initGroovyLog();
assertions.clear();
String projectFile = getProjectFile();
WsdlProject project = (WsdlProject) ProjectFactoryRegistry.getProjectFactory("wsdl").createNew(projectFile,
getProjectPassword());
if (project.isDisabled()) {
throw new Exception("Failed to load SoapUI project file [" + projectFile + "]");
}
initProject(project);
ensureOutputFolder(project);
if (this.printAlertSiteReport) {
testCaseRunLogReport = new TestCaseRunLogReport(getAbsoluteOutputFolder(project));
}
log.info("Running SoapUI tests in project [" + project.getName() + "]");
long startTime = System.nanoTime();
List<TestCase> testCasesToRun = new ArrayList<TestCase>();
// validate testSuite argument
if (testSuite != null && project.getTestSuiteByName(testSuite) == null) {
throw new Exception("TestSuite with name [" + testSuite + "] is missing in Project [" + project.getName()
+ "]");
}
// start by listening to all testcases.. (since one testcase can call
// another)
for (int c = 0; c < project.getTestSuiteCount(); c++) {
TestSuite suite = project.getTestSuiteAt(c);
for (int i = 0; i < suite.getTestCaseCount(); i++) {
TestCase tc = suite.getTestCaseAt(i);
if ((testSuite == null || suite.getName().equals(testSuite)) && testCase != null
&& tc.getName().equals(testCase)) {
testCasesToRun.add(tc);
}
addListeners(tc);
}
}
try {
// validate testSuite argument
if (testCase != null && testCasesToRun.size() == 0) {
if (testSuite == null) {
throw new Exception("TestCase with name [" + testCase + "] is missing in Project [" + project.getName()
+ "]");
} else {
throw new Exception("TestCase with name [" + testCase + "] in TestSuite [" + testSuite
+ "] is missing in Project [" + project.getName() + "]");
}
}
// decide what to run
if (testCasesToRun.size() > 0) {
for (TestCase testCase : testCasesToRun) {
runTestCase((WsdlTestCase) testCase);
}
} else if (testSuite != null) {
WsdlTestSuite ts = project.getTestSuiteByName(testSuite);
if (ts == null) {
throw new Exception("TestSuite with name [" + testSuite + "] not found in project");
} else {
runSuite(ts);
}
} else {
runProject(project);
}
long timeTaken = (System.nanoTime() - startTime) / 1000000;
if (printReport) {
printReport(timeTaken);
}
exportReports(project);
if (saveAfterRun && !project.isRemote()) {
try {
project.save();
} catch (Throwable t) {
log.error("Failed to save project", t);
}
}
if ((assertions.size() > 0 || failedTests.size() > 0) && !ignoreErrors) {
throwFailureException();
}
return true;
} finally {
for (int c = 0; c < project.getTestSuiteCount(); c++) {
TestSuite suite = project.getTestSuiteAt(c);
for (int i = 0; i < suite.getTestCaseCount(); i++) {
TestCase tc = suite.getTestCaseAt(i);
removeListeners(tc);
}
}
}
}
protected void removeListeners(TestCase tc) {
tc.removeTestRunListener(this);
if (junitReport) {
tc.removeTestRunListener(reportCollector);
}
}
protected void runProject(WsdlProject project) {
// add listener for counting..
InternalProjectRunListener projectRunListener = new InternalProjectRunListener();
project.addProjectRunListener(projectRunListener);
try {
log.info(("Running Project [" + project.getName() + "], runType = " + project.getRunType()));
WsdlProjectRunner runner = project.run(new StringToObjectMap(), false);
log.info("Project [" + project.getName() + "] finished with status [" + runner.getStatus() + "] in "
+ runner.getTimeTaken() + "ms");
} catch (Exception e) {
e.printStackTrace();
} finally {
project.removeProjectRunListener(projectRunListener);
}
}
protected void initProject(WsdlProject project) throws Exception {
initProjectProperties(project);
}
protected void exportReports(WsdlProject project) throws Exception {
if (junitReport) {
exportJUnitReports(reportCollector, getAbsoluteOutputFolder(project), project);
}
}
protected void addListeners(TestCase tc) {
tc.addTestRunListener(this);
if (junitReport) {
tc.addTestRunListener(reportCollector);
}
if (printAlertSiteReport) {
tc.addTestRunListener(testCaseRunLogReport);
}
}
protected void throwFailureException() throws Exception {
StringBuffer buf = new StringBuffer();
for (int c = 0; c < assertions.size(); c++) {
TestAssertion assertion = assertions.get(c);
Assertable assertable = assertion.getAssertable();
if (assertable instanceof WsdlTestStep) {
failedTests.remove(((WsdlTestStep) assertable).getTestCase());
}
buf.append(assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n");
buf.append(Arrays.toString(assertion.getErrors()) + "\n");
WsdlTestStepResult result = assertionResults.get(assertion);
StringWriter stringWriter = new StringWriter();
PrintWriter writer = new PrintWriter(stringWriter);
result.writeTo(writer);
buf.append(stringWriter.toString());
}
while (!failedTests.isEmpty()) {
buf.append("TestCase [" + failedTests.remove(0).getName() + "] failed without assertions\n");
}
throw new Exception(buf.toString());
}
public boolean isExportAll() {
return exportAll;
}
public void exportJUnitReports(JUnitReportCollector collector, String folder, WsdlProject project)
throws Exception {
collector.saveReports(folder == null ? "" : folder);
}
public void printReport(long timeTaken) {
System.out.println();
System.out.println("SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary");
System.out.println("-----------------------------");
System.out.println("Time Taken: " + timeTaken + "ms");
System.out.println("Total TestSuites: " + testSuiteCount);
System.out.println("Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
System.out.println("Total TestSteps: " + testStepCount);
System.out.println("Total Request Assertions: " + testAssertionCount);
System.out.println("Total Failed Assertions: " + assertions.size());
System.out.println("Total Exported Results: " + exportCount);
}
/**
* Run tests in the specified TestSuite
*
* @param suite the TestSuite to run
*/
protected void runSuite(WsdlTestSuite suite) {
try {
log.info(("Running TestSuite [" + suite.getName() + "], runType = " + suite.getRunType()));
WsdlTestSuiteRunner runner = suite.run(new StringToObjectMap(), false);
log.info("TestSuite [" + suite.getName() + "] finished with status [" + runner.getStatus() + "] in "
+ (runner.getTimeTaken()) + "ms");
} catch (Exception e) {
e.printStackTrace();
} finally {
testSuiteCount++;
}
}
/**
* Runs the specified TestCase
*
* @param testCase the testcase to run
* @param context
*/
protected void runTestCase(WsdlTestCase testCase) {
try {
log.info("Running TestCase [" + testCase.getName() + "]");
WsdlTestCaseRunner runner = testCase.run(new StringToObjectMap(), false);
log.info("TestCase [" + testCase.getName() + "] finished with status [" + runner.getStatus() + "] in "
+ (runner.getTimeTaken()) + "ms");
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Sets the testcase to run
*
* @param testCase the testcase to run
*/
public void setTestCase(String testCase) {
this.testCase = testCase;
}
/**
* Sets the TestSuite to run. If not set all TestSuites in the specified
* project file are run
*
* @param testSuite the testSuite to run.
*/
public void setTestSuite(String testSuite) {
this.testSuite = testSuite;
}
@Override
public void beforeRun(TestCaseRunner testRunner, TestCaseRunContext runContext) {
log.info("Running SoapUI testcase [" + testRunner.getTestCase().getName() + "]");
}
@Override
public void beforeStep(TestCaseRunner testRunner, TestCaseRunContext runContext, TestStep currentStep) {
super.beforeStep(testRunner, runContext, currentStep);
if (currentStep != null) {
log.info("running step [" + currentStep.getName() + "]");
}
}
@Override
public void afterStep(TestCaseRunner testRunner, TestCaseRunContext runContext, TestStepResult result) {
super.afterStep(testRunner, runContext, result);
TestStep currentStep = runContext.getCurrentStep();
if (currentStep instanceof Assertable) {
Assertable requestStep = (Assertable) currentStep;
for (int c = 0; c < requestStep.getAssertionCount(); c++) {
TestAssertion assertion = requestStep.getAssertionAt(c);
log.info("Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
if (assertion.getStatus() == AssertionStatus.FAILED) {
for (AssertionError error : assertion.getErrors()) {
log.error("ASSERTION FAILED -> " + error.getMessage());
}
assertions.add(assertion);
assertionResults.put(assertion, (WsdlTestStepResult) result);
}
testAssertionCount++;
}
}
String countPropertyName = currentStep.getName() + " run count";
Long count = (Long) runContext.getProperty(countPropertyName);
if (count == null) {
count = new Long(0);
}
runContext.setProperty(countPropertyName, new Long(count.longValue() + 1));
if (result.getStatus() == TestStepStatus.FAILED || exportAll) {
try {
String exportSeparator = System.getProperty(SOAPUI_EXPORT_SEPARATOR, "-");
TestCase tc = currentStep.getTestCase();
String nameBase = StringUtils.createFileName(tc.getTestSuite().getName(), '_') + exportSeparator
+ StringUtils.createFileName(tc.getName(), '_') + exportSeparator
+ StringUtils.createFileName(currentStep.getName(), '_') + "-" + count.longValue() + "-"
+ result.getStatus();
WsdlTestCaseRunner callingTestCaseRunner = (WsdlTestCaseRunner) runContext
.getProperty("#CallingTestCaseRunner#");
if (callingTestCaseRunner != null) {
WsdlTestCase ctc = callingTestCaseRunner.getTestCase();
WsdlRunTestCaseTestStep runTestCaseTestStep = (WsdlRunTestCaseTestStep) runContext
.getProperty("#CallingRunTestCaseStep#");
nameBase = StringUtils.createFileName(ctc.getTestSuite().getName(), '_') + exportSeparator
+ StringUtils.createFileName(ctc.getName(), '_') + exportSeparator
+ StringUtils.createFileName(runTestCaseTestStep.getName(), '_') + exportSeparator
+ StringUtils.createFileName(tc.getTestSuite().getName(), '_') + exportSeparator
+ StringUtils.createFileName(tc.getName(), '_') + exportSeparator
+ StringUtils.createFileName(currentStep.getName(), '_') + "-" + count.longValue() + "-"
+ result.getStatus();
}
String absoluteOutputFolder = getAbsoluteOutputFolder(ModelSupport.getModelItemProject(tc));
String fileName = absoluteOutputFolder + File.separator + nameBase + ".txt";
if (result.getStatus() == TestStepStatus.FAILED) {
log.error(currentStep.getName() + " failed, exporting to [" + fileName + "]");
}
new File(fileName).getParentFile().mkdirs();
PrintWriter writer = new PrintWriter(fileName);
result.writeTo(writer);
writer.close();
// write attachments
if (result instanceof MessageExchange) {
Attachment[] attachments = ((MessageExchange) result).getResponseAttachments();
if (attachments != null && attachments.length > 0) {
for (int c = 0; c < attachments.length; c++) {
fileName = nameBase + "-attachment-" + (c + 1) + ".";
Attachment attachment = attachments[c];
String contentType = attachment.getContentType();
if (!"application/octet-stream".equals(contentType) && contentType != null
&& contentType.indexOf('/') != -1) {
fileName += contentType.substring(contentType.lastIndexOf('/') + 1);
} else {
fileName += "dat";
}
fileName = absoluteOutputFolder + File.separator + fileName;
FileOutputStream outFile = new FileOutputStream(fileName);
Tools.writeAll(outFile, attachment.getInputStream());
outFile.close();
}
}
}
exportCount++;
} catch (Exception e) {
log.error("Error saving failed result: " + e, e);
}
}
testStepCount++;
}
@Override
public void afterRun(TestCaseRunner testRunner, TestCaseRunContext runContext) {
log.info("Finished running SoapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: "
+ testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus());
if (testRunner.getStatus() == Status.FAILED) {
failedTests.add(testRunner.getTestCase());
}
testCaseCount++;
}
protected class InternalProjectRunListener extends ProjectRunListenerAdapter {
@Override
public void afterTestSuite(ProjectRunner projectRunner, ProjectRunContext runContext, TestSuiteRunner testRunner) {
testSuiteCount++;
}
}
public String getTestSuite() {
return testSuite;
}
public String getTestCase() {
return testCase;
}
public boolean isJUnitReport() {
return junitReport;
}
public boolean isIgnoreErrors() {
return ignoreErrors;
}
public void setIgnoreErrors(boolean ignoreErrors) {
this.ignoreErrors = ignoreErrors;
}
public boolean isPrintReport() {
return printReport;
}
public boolean isSaveAfterRun() {
return saveAfterRun;
}
public List<TestCase> getFailedTests() {
return failedTests;
}
public List<TestAssertion> getAssertions() {
return assertions;
}
public boolean isJunitReport() {
return junitReport;
}
public int getExportCount() {
return exportCount;
}
public void setExportCount(int exportCount) {
this.exportCount = exportCount;
}
public Map<TestAssertion, WsdlTestStepResult> getAssertionResults() {
return assertionResults;
}
public int getTestStepCount() {
return testStepCount;
}
public void setTestStepCount(int testStepCount) {
this.testStepCount = testStepCount;
}
}