Examples of Pipeline


Examples of LONI.tree.workflow.Pipeline

      case GALAXY:
        outputString = GalaxySpecification.getJSONGenerator().generate(
            g);
        break;
      case LONI:
        Pipeline p;

        // don't import; use fully-qualified name to avoid collision
        // with local converter
        Galaxy.Visitor.Webservice.GalaxyToLoniWorkflowConverter foo = new Galaxy.Visitor.Webservice.GalaxyToLoniWorkflowConverter();
        Printer.log("Using Webservice Galaxy Converter");
        p = (Pipeline) foo.visit(g);

        outputString = LoniSpecification.getXMLGenerator().generate(p);
        break;
      default:
        cantConvert();
      }
      break;
    }
    case TAVERNA: {
      Taverna.Tree.Workflow t = null;
      try {
        t = TavernaSpecification.getXMLParser().parse(
            new File(ConverterConfig.INPUT_PATH));
      } catch (FileNotFoundException e) {
        Printer.log("What, FileNotFoundException not caught by configureInput?");
      }

      switch (ConverterConfig.OUTPUT_FORMAT) {
      case TAVERNA:
        outputString = TavernaSpecification.getXMLGenerator().generate(
            t);
        break;
      case LONI:
        TavernaToLoniConverter foo = new TavernaToLoniConverter();
        Pipeline p = (Pipeline) foo.visit(t);
        outputString = LoniSpecification.getXMLGenerator().generate(p);
        break;
      default:
        cantConvert();
      }
      break;
    }
    case LONI: {
      Pipeline p = null;
      try {
        p = LoniSpecification.getXMLParser().parse(
            new File(ConverterConfig.INPUT_PATH));
      } catch (FileNotFoundException e) {
        Printer.log("What, FileNotFoundException not caught by configureInput?");
View Full Code Here

Examples of codec.gen.configuration.Pipeline

        {
            log.debug("processPipeline(Configuration, File) - start");
        }

        boolean succesful = false;
        Pipeline pipeline = (Pipeline)configuration.getPipeline();
        File sourceFile   = new File(pipeline.getSource().getFile());

        InputSource input = null;

        if (pipeline.getSource().getType().equals("ASN1"))
        {
            log.info(
                "Processing ASN.1 file: " + sourceFile.getAbsolutePath());

            SimpleNode root = Asn1Parser.parse(sourceFile);
            if (root == null)
            {
                if (log.isEnabled(LogLevel.DEBUG))
                {
                    log.debug("processPipeline(Configuration, File) - end");
                }
                return false;
            }

            // Set up the input stream
            input = new InputSource(new StringReader(createXMLfromAST(root)));
        }
        else
        {
            log.info("Processing XML file: " + sourceFile.getAbsolutePath());
            // Set up the input stream
            try
            {
                input = new InputSource(new FileReader(sourceFile));
            }
            catch (FileNotFoundException e)
            {
                log.caught(
                    LogLevel.ERROR,
                    "processPipeline(Configuration, File) :" + e.getMessage(),
                    e);
                return false;
            }
        }

        try
        {
            // Set up to read the input file
            SAXParserFactory spf = SAXParserFactory.newInstance();
            spf.setNamespaceAware(true);
            SAXParser parser = spf.newSAXParser();
            XMLReader reader = parser.getXMLReader();

            // Create the filters
            SAXTransformerFactory stf =
                (SAXTransformerFactory)TransformerFactory.newInstance();

            URIResolver resolver =
                new ClasspathURIResolver(stf.getURIResolver());
            stf.setURIResolver(resolver);

            net.sf.saxon.Filter filter = null;

            XMLReader parent           = reader;

            for (int i = 0; i < pipeline.getFilter().size(); i++)
            {
                // extract information from pipeline structure
                PipelineType.FilterType filterType =
                    ((PipelineType.FilterType)pipeline.getFilter().get(i));

                if (!filterType.isDisabled())
                {
                    log.info(
                        "attach filter: " + filterType.getId() + " ("
                        + filterType.getXsl() + ")");

                    Source xslSource =
                        resolver.resolve(filterType.getXsl(), null);

                    filter = (net.sf.saxon.Filter)stf.newXMLFilter(xslSource);
                    filter.setParent(parent);

                    parent = filter;
                    // set XSL parameter; pipeline configuration
                    filter.getTransformer().setParameter(
                        "configFile",
                        replacePathSeparator(configurationFile));
                    // set XSL parameter; filterID
                    filter.getTransformer().setParameter(
                        "filterID",
                        filterType.getId());
                    // set XSL parameter; filterdescription
                    filter.getTransformer().setParameter(
                        "filterDescription",
                        filterType.getDescription());
                }
            }

            // Set up the result stream
            Writer writer = null;
            if (pipeline.getSink() != null)
            {
                File ouputFile = new File(pipeline.getSink().getFile());
                writer = new FileWriter(ouputFile);
            }
            else
            {
                writer = new StringWriter();
            }

            StreamResult result = new StreamResult(writer);

            // Set up the transformer to process the SAX events generated
            // by the last filter in the chain
            Transformer transformer = stf.newTransformer();

            // set outputproperties
            transformer.setOutputProperty(OutputKeys.ENCODING, "ISO-8859-1");
            transformer.setOutputProperty(OutputKeys.METHOD, "xml");
            transformer.setOutputProperty(
                OutputKeys.OMIT_XML_DECLARATION,
                "no");
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
            transformer.setOutputProperty(OutputKeys.MEDIA_TYPE, "text/xml");
            transformer.setOutputProperty(OutputKeys.STANDALONE, "yes");


            log.info("Processing pipeline");
            SAXSource transformSource = new SAXSource(parent, input);
            transformer.transform(transformSource, result);

            if (log.isEnabled(LogLevel.DEBUG))
            {
                if (pipeline.getSink() != null)
                {
                    log.debug(
                        "processPipeline(Configuration, File) output:\n"
                        + pipeline.getSink().getFile());
                }
                else
                {
                    log.debug(
                        "processPipeline(Configuration, File) output:\n"
View Full Code Here

Examples of com.alibaba.citrus.service.pipeline.Pipeline

        // no body/conditon
        assertEquals(str, valve.toString());

        // with body/condition
        Pipeline pipeline = createPipeline(new LogValve(), new LogValve(), new LogValve());
        valve.setCondition(new JexlCondition("1==2"));
        valve.setLoopBody(pipeline);

        str = "";
        str += "WhileLoopValve {\n";
View Full Code Here

Examples of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline

    public void start() {
        if (running) {
            return;
        }
        // 获取destination/filter参数
        Pipeline pipeline = configClientService.findPipeline(pipelineId);
        filter = makeFilterExpression(pipeline);
        destination = pipeline.getParameters().getDestinationName();
        batchSize = pipeline.getParameters().getMainstemBatchsize();
        batchTimeout = pipeline.getParameters().getBatchTimeout();
        ddlSync = pipeline.getParameters().getDdlSync();
        if (pipeline.getParameters().getDumpSelector() != null) {
            dump = pipeline.getParameters().getDumpSelector();
        }

        if (pipeline.getParameters().getDumpSelectorDetail() != null) {
            dumpDetail = pipeline.getParameters().getDumpSelectorDetail();
        }

        canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

            public CanalInstance generate(String destination) {
                Canal canal = canalConfigClient.findCanal(destination);
                final OtterAlarmHandler otterAlarmHandler = new OtterAlarmHandler();
                otterAlarmHandler.setPipelineId(pipelineId);
                OtterContextLocator.autowire(otterAlarmHandler); // 注入一下spring资源
                // 设置下slaveId,保证多个piplineId下重复引用时不重复
                long slaveId = 10000;// 默认基数
                if (canal.getCanalParameter().getSlaveId() != null) {
                    slaveId = canal.getCanalParameter().getSlaveId();
                }
                canal.getCanalParameter().setSlaveId(slaveId + pipelineId);
                canal.getCanalParameter().setDdlIsolation(ddlSync);

                CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, filter) {

                    protected CanalHAController initHaController() {
                        HAMode haMode = parameters.getHaMode();
                        if (haMode.isMedia()) {
                            return new MediaHAController(parameters.getMediaGroup(),
                                parameters.getDbUsername(),
                                parameters.getDbPassword(),
                                parameters.getDefaultDatabaseName());
                        } else {
                            return super.initHaController();
                        }
                    }

                    protected void startEventParserInternal(CanalEventParser parser, boolean isGroup) {
                        super.startEventParserInternal(parser, isGroup);

                        if (eventParser instanceof MysqlEventParser) {
                            MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
                            CanalHAController haController = mysqlEventParser.getHaController();

                            if (haController instanceof MediaHAController) {
                                if (isGroup) {
                                    throw new CanalException("not support group database use media HA");
                                }

                                ((MediaHAController) haController).setCanalHASwitchable(mysqlEventParser);
                            }

                            if (!haController.isStart()) {
                                haController.start();
                            }

                            // 基于media的Ha,直接从tddl中获取数据库信息
                            if (haController instanceof MediaHAController) {
                                AuthenticationInfo authenticationInfo = ((MediaHAController) haController).getAvailableAuthenticationInfo();
                                ((MysqlEventParser) eventParser).setMasterInfo(authenticationInfo);
                            }
                        }
                    }

                };
                instance.setAlarmHandler(otterAlarmHandler);

                CanalEventSink eventSink = instance.getEventSink();
                if (eventSink instanceof AbstractCanalEventSink) {
                    handler = new OtterDownStreamHandler();
                    handler.setPipelineId(pipelineId);
                    handler.setDetectingIntervalInSeconds(canal.getCanalParameter().getDetectingIntervalInSeconds());
                    OtterContextLocator.autowire(handler); // 注入一下spring资源
                    ((AbstractCanalEventSink) eventSink).addHandler(handler, 0); // 添加到开头
                    handler.start();
                }

                return instance;
            }
        });
        canalServer.start();

        canalServer.start(destination);
        this.clientIdentity = new ClientIdentity(destination, pipeline.getParameters().getMainstemClientId(), filter);
        canalServer.subscribe(clientIdentity);// 发起一次订阅

        running = true;
    }
View Full Code Here

Examples of com.cloudera.crunch.Pipeline

public class SortCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(SortCrunchTest.class);
    PCollection<String> records = pipeline.readTextFile("sort/A");
   
    PCollection<Pair<Integer, Integer>> pairs = records.parallelDo(new DoFn<String, Pair<Integer, Integer>>() {
      @Override
      public void process(String input, Emitter<Pair<Integer, Integer>> emitter) {
        Iterator<String> split = Splitter.on('\t').split(input).iterator();
        String l = split.next();
        String r = split.next();
        emitter.emit(Pair.of(Integer.parseInt(l), Integer.parseInt(r)));
      }
    }, pairs(ints(), ints()));
   
    PCollection<Pair<Integer, Integer>> sorted = Sort.sortPairs(pairs, by(1, ASCENDING), by(2, DESCENDING));
   
    pipeline.writeTextFile(sorted, "output-sorted");
    pipeline.run();
  }
View Full Code Here

Examples of com.findwise.hydra.Pipeline

            MongoDBConnectionConfig conf = new MongoDBConnectionConfig(pipelineConfig.getPipelineName(), cmd.getHost(), "", "");
            MongoConnector mdc =new MongoConnector(conf.getConfiguration());
            mdc.connect();

            Pipeline pipeline = mdc.getPipelineReader().getPipeline();
            for (Stage stage : stages) {
                if (cmd.getStageNames() != null) {
                    if (cmd.getStageNames().contains(stage.getName())) {
                        log.info("Preparing to upload stage, " + stage.getName());
                        StageGroup g = new StageGroup(stage.getName());
                        g.addStage(stage);
                        pipeline.addGroup(g);
                    }
                } else {
                    log.info("Preparing to upload stage, " + stage.getName());
                    StageGroup g = new StageGroup(stage.getName());
                    g.addStage(stage);
                    pipeline.addGroup(g);
                }
            }
            log.info("Uploading stages");
            log.info(pipeline.toString());
            mdc.getPipelineWriter().write(pipeline);
        }
    }
View Full Code Here

Examples of com.impetus.labs.korus.addons.constructs.pipeline.Pipeline

    long initialTime, finalTime;
    initialTime = System.currentTimeMillis();

    // Create the object of the pipeline.
    Pipeline pipeline = new Pipeline("pipeline test");

    // Create objects Tasks which extend PipelineTasks
    ReadTask task1 = new ReadTask();
    TransformTask task2 = new TransformTask();
    WriteTask task3 = new WriteTask();

    // Add these tasks to the pipeline
    pipeline.add("readTask", task1);
    pipeline.add("transformTask", task2);
    pipeline.add("writeTask", task3);

    // Join these tasks in order to know the order of execution of the tasks
    pipeline.join(task1, task2);
    pipeline.join(task2, task3);

    // Execute the Pipeline
    pipeline.execute();

    // To wait till execution lasts. Skip this line pipeline.cleanUp()
    // if you do not want to
    // wait for the execution of pipeline to finish.
    pipeline.getResult();

    finalTime = System.currentTimeMillis();
    System.out.println("Time Taken by ParallelRTW: "
        + (finalTime - initialTime));

    // shutdown when task is over
    pipeline.cleanUp();

  }
View Full Code Here

Examples of com.linkedin.helix.controller.pipeline.Pipeline

    synchronized (GenericHelixController.class)
    {
      PipelineRegistry registry = new PipelineRegistry();

      // cluster data cache refresh
      Pipeline dataRefresh = new Pipeline();
      dataRefresh.addStage(new ReadClusterDataStage());

      // rebalance pipeline
      Pipeline rebalancePipeline = new Pipeline();
      rebalancePipeline.addStage(new ResourceComputationStage());
      rebalancePipeline.addStage(new CurrentStateComputationStage());
      rebalancePipeline.addStage(new BestPossibleStateCalcStage());
      rebalancePipeline.addStage(new MessageGenerationPhase());
      rebalancePipeline.addStage(new MessageSelectionStage());
      rebalancePipeline.addStage(new MessageThrottleStage());
      rebalancePipeline.addStage(new TaskAssignmentStage());

      // external view generation
      Pipeline externalViewPipeline = new Pipeline();
      externalViewPipeline.addStage(new ExternalViewComputeStage());

      // backward compatibility check
      Pipeline liveInstancePipeline = new Pipeline();
      liveInstancePipeline.addStage(new CompatibilityCheckStage());

      registry.register("idealStateChange", dataRefresh, rebalancePipeline);
      registry.register("currentStateChange",
                        dataRefresh,
                        rebalancePipeline,
View Full Code Here

Examples of com.rackspace.cloud.api.docs.pipeline.Pipeline

import java.util.Map;

public class CalabashHelper {
    private static Source run(final Log log, final String pipelineURI, final InputSource inputSource, final Map<String, Object> map) throws FileNotFoundException {
        XProcMessageListener messageListener = log != null ? new MavenXProcMessageListener(log) : null;
        Pipeline pipeline = new CalabashPipelineBuilder(false, true, messageListener).build(pipelineURI);

//        <c:param-set xmlns:c="http://www.w3.org/ns/xproc-step">
//            <c:param name="username" namespace="" value="user"/>
//            <c:param name="host" namespace="" value="http://example.com/"/>
//            <c:param name="password" namespace="" value="pass"/>
//        </c:param-set>
       
        StringBuffer strBuff = new StringBuffer("<c:param-set xmlns:c=\"http://www.w3.org/ns/xproc-step\">");
        strBuff.append("");
       
       
        if(null!=map){
            for (Map.Entry<String, Object> entry : map.entrySet()) {
                String rawValue;
                if (entry.getValue() instanceof File) {
                    rawValue = ((File)entry.getValue()).toURI().toString();
                } else if (entry.getValue() instanceof URI || entry.getValue() instanceof String) {
                    rawValue = entry.getValue().toString();
                } else if (entry.getValue() instanceof URL) {
                    rawValue = ((URL)entry.getValue()).toExternalForm();
                } else if (entry.getValue() != null) {
                    throw new UnsupportedOperationException(String.format("The map cannot contain values of type %s.", entry.getValue().getClass()));
                } else {
                    // ignore nulls
                    continue;
                }

                strBuff
                    .append("<c:param name=\"")
                    .append(escapeXmlAttribute(entry.getKey()))
                    .append("\" namespace=\"\" value=\"")
                    .append(escapeXmlAttribute(rawValue))
                    .append("\"/>");
            }
        }

        strBuff.append("</c:param-set>");
        String params=strBuff.toString();
        final InputStream paramsStream = new ByteArrayInputStream(params.getBytes());
       
        //System.out.println("~!~!~!~!~!~!~!~!~!Sending: \n"+params+"\n");
        List<PipelineInput<?>> pipelineInputs = new ArrayList<PipelineInput<?>>() {{
            add(PipelineInput.port("source", inputSource));
            add(PipelineInput.port("parameters", new InputSource(paramsStream)));
        }};
       
        pipeline.run(pipelineInputs);
        List<Source> sources = pipeline.getResultPort("result"); // result of xinclude;

        return sources.get(0);
    }
View Full Code Here

Examples of com.sun.enterprise.web.connector.grizzly.Pipeline

        }
            
        String token = getContextRoot();
       
        // Lazy instanciation
        Pipeline pipeline = pipelines.get(token);
        if (pipeline == null){
            pipeline = applyRule(token);
            pipelines.put(token,pipeline);
        }
               
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.