Package com.dp.nebula.wormhole.common.config

Examples of com.dp.nebula.wormhole.common.config.JobConf


public class ParseXMLUtilTest {

  @Test
  public void testLoadJobConf() {
    String fileName = "src/test/resources/wormhole_hivereader_to_hdfswriter_test.xml";
    JobConf jobConf = ParseXMLUtil.loadJobConf(fileName);
    assertNotNull(jobConf);
    assertEquals("hivereader_to_hdfswriter_job", jobConf.getId());
   
    JobPluginConf readerConf =  jobConf.getReaderConf();
    List<JobPluginConf> writerConf = jobConf.getWriterConfs();
   
    assertEquals("hivereader", readerConf.getPluginName());
    IParam readerPluginParam = readerConf.getPluginParam();
    assertNotNull(readerPluginParam);
    assertTrue(readerPluginParam instanceof IParam);
View Full Code Here


    // return usage information
    else {
      s_logger.error("Usage: ./wormhole.sh job.xml .");
      System.exit(JobStatus.FAILED.getStatus());
    }
    JobConf jobConf = null;
    IParam engineConf = null;
    Map<String, IParam> pluginConfs = null;
    try {
      // read configurations from XML for engine & plugins
      jobConf = ParseXMLUtil.loadJobConf(jobDescriptionXML);
View Full Code Here

    //return usage information
    else{
      s_logger.error("Usage: ./wormhole.sh job.xml .");
      System.exit(JobStatus.FAILED.getStatus());
    }
    JobConf jobConf = null;
    IParam engineConf = null;
    Map<String, IParam> pluginConfs = null;
    try{
      //read configurations from XML for engine & plugins
      jobConf = ParseXMLUtil.loadJobConf(jobDescriptionXML);
View Full Code Here

   * @return a job configure instance with the specified job configuration
   *         file
   */
  @SuppressWarnings("unchecked")
  public static JobConf loadJobConf(String filename) {
    JobConf job = new JobConf();
    Document document = null;

    try {
      String xml = FileUtils
          .readFileToString(new File(filename), "UTF-8");
      xml = StringUtil.replaceEnvironmentVariables(xml);
      document = DocumentHelper.parseText(xml);
    } catch (IOException e) {
      LOG.error(String.format("WormHole can't find job conf file: %s.",
          filename));
    } catch (DocumentException e) {
      LOG.error(String.format("Parse %s to document failed.", filename));
    }

    String xpath = "/job";

    Element jobE = (Element) document.selectSingleNode(xpath);
    String jobId = jobE.attributeValue("id", "WormHole_id_not_found")
        .trim();
    job.setId(jobId);

    JobPluginConf readerJobConf = new JobPluginConf();
    Element readerE = (Element) jobE.selectSingleNode(xpath + "/reader");
    Element readerPluginE = (Element) readerE.selectSingleNode("plugin");
    String readerId = readerE.attributeValue("id");
    String readerName = readerPluginE.getStringValue().trim().toLowerCase();
    readerJobConf.setPluginName(readerName);
    readerJobConf.setId(readerId == null ? "reader-id-" + readerName
        : readerId.trim());
    Map<String, String> readerPluginParamMap = getParamMap(readerE);

    IParam readerPluginParam = new DefaultParam(readerPluginParamMap);
    readerJobConf.setPluginParam(readerPluginParam);

    List<JobPluginConf> writerJobConfs = new ArrayList<JobPluginConf>();
    List<Element> writerEs = (List<Element>) document.selectNodes(xpath
        + "/writer");

    for (Element writerE : writerEs) {
      JobPluginConf writerPluginConf = new JobPluginConf();

      Element writerPluginE = (Element) writerE
          .selectSingleNode("plugin");
      String writerName = writerPluginE.getStringValue().trim()
          .toLowerCase();
      String writerId = writerE.attributeValue("id");
      writerPluginConf.setPluginName(writerName);
      writerPluginConf.setId(writerId == null ? "writer-id-"
          + writerEs.indexOf(writerE) + "-" + writerName : writerId
          .trim());

      Map<String, String> writerPluginParamMap = getParamMap(writerE);

      IParam writerPluginParam = new DefaultParam(writerPluginParamMap);

      writerPluginConf.setPluginParam(writerPluginParam);
      writerJobConfs.add(writerPluginConf);
    }

    job.setReaderConf(readerJobConf);
    job.setWriterConfs(writerJobConfs);

    return job;
  }
View Full Code Here

TOP

Related Classes of com.dp.nebula.wormhole.common.config.JobConf

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.