Package com.alibaba.otter.shared.arbitrate.model

Examples of com.alibaba.otter.shared.arbitrate.model.EtlEventData


                    // 添加到待响应的buffer列表,不需要await termin信号,因为没启动过s/e/t/l流程
                    batchBuffer.put(new BatchTermin(gotMessage.getId(), false));
                    continue;
                }

                final EtlEventData etlEventData = arbitrateEventService.selectEvent().await(pipelineId);
                if (rversion.get() != startVersion) {// 说明存在过变化,中间出现过rollback,需要丢弃该数据
                    logger.warn("rollback happend , should skip this data and get new message.");
                    canStartSelector.get();// 确认一下rollback是否完成
                    gotMessage = otterSelector.selector();// 这时不管有没有数据,都需要执行一次s/e/t/l
                }

                final Message message = gotMessage;
                final BatchTermin batchTermin = new BatchTermin(message.getId(), etlEventData.getProcessId());
                batchBuffer.put(batchTermin); // 添加到待响应的buffer列表
                Runnable task = new Runnable() {

                    public void run() {
                        // 设置profiling信息
                        boolean profiling = isProfiling();
                        Long profilingStartTime = null;
                        if (profiling) {
                            profilingStartTime = System.currentTimeMillis();
                        }

                        MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipelineId));
                        String currentName = Thread.currentThread().getName();
                        Thread.currentThread().setName(createTaskName(pipelineId, "SelectWorker"));
                        try {
                            pipeline = configClientService.findPipeline(pipelineId);
                            List<EventData> eventData = message.getDatas();
                            long startTime = etlEventData.getStartTime();
                            if (!CollectionUtils.isEmpty(eventData)) {
                                startTime = eventData.get(0).getExecuteTime();
                            }

                            Channel channel = configClientService.findChannelByPipelineId(pipelineId);
                            RowBatch rowBatch = new RowBatch();
                            // 构造唯一标识
                            Identity identity = new Identity();
                            identity.setChannelId(channel.getId());
                            identity.setPipelineId(pipelineId);
                            identity.setProcessId(etlEventData.getProcessId());
                            rowBatch.setIdentity(identity);
                            // 进行数据合并
                            for (EventData data : eventData) {
                                rowBatch.merge(data);
                            }

                            long nextNodeId = etlEventData.getNextNid();
                            List<PipeKey> pipeKeys = rowDataPipeDelegate.put(new DbBatch(rowBatch), nextNodeId);
                            etlEventData.setDesc(pipeKeys);
                            etlEventData.setNumber((long) eventData.size());
                            etlEventData.setFirstTime(startTime); // 使用原始数据的第一条
                            etlEventData.setBatchId(message.getId());

                            if (profiling) {
                                Long profilingEndTime = System.currentTimeMillis();
                                stageAggregationCollector.push(pipelineId,
                                                               StageType.SELECT,
                                                               new AggregationItem(profilingStartTime, profilingEndTime));
                            }
                            arbitrateEventService.selectEvent().single(etlEventData);
                        } catch (Throwable e) {
                            if (!isInterrupt(e)) {
                                logger.error(String.format("[%s] selectwork executor is error! data:%s", pipelineId,
                                                           etlEventData), e);
                                sendRollbackTermin(pipelineId, e);
                            } else {
                                logger.info(String.format("[%s] selectwork executor is interrrupt! data:%s",
                                                          pipelineId, etlEventData), e);
                            }
                        } finally {
                            Thread.currentThread().setName(currentName);
                            MDC.remove(OtterConstants.splitPipelineLogFileKey);
                        }
                    }
                };

                // 构造pending任务,可在关闭线程时退出任务
                SetlFuture extractFuture = new SetlFuture(StageType.SELECT, etlEventData.getProcessId(), pendingFuture,
                                                          task);
                executorService.execute(extractFuture);

            } catch (Throwable e) {
                if (!isInterrupt(e)) {
View Full Code Here


            // 根据pipelineId+processId构造对应的path
            String path = StagePathUtils.getSelectStage(pipelineId, processId);

            try {
                byte[] data = zookeeper.readData(path);
                EtlEventData eventData = JsonUtils.unmarshalFromByte(data, EtlEventData.class);

                Node node = LoadBalanceFactory.getNextTransformNode(pipelineId);// 获取下一个处理节点信息
                if (node == null) {// 没有后端节点
                    // TerminEventData termin = new TerminEventData();
                    // termin.setPipelineId(pipelineId);
                    // termin.setType(TerminType.ROLLBACK);
                    // termin.setCode("no_node");
                    // termin.setDesc(MessageFormat.format("pipeline[{}] extract stage has no node!", pipelineId));
                    // terminEvent.single(termin);
                    throw new ArbitrateException("Extract_single", "no next node");
                } else {
                    eventData.setNextNid(node.getId());
                    return eventData;// 只有这一条路返回
                }
            } catch (ZkNoNodeException e) {
                logger.error("pipeline[{}] processId[{}] is invalid , retry again", pipelineId, processId);
                return await(pipelineId);// /出现节点不存在,说明出现了error情况,递归调用重新获取一次
View Full Code Here

    public void dispose() {
        ArbitrateFactory.destory(pipelineId);
    }

    protected EtlEventData getData(Long nid) {
        EtlEventData data = new EtlEventData();
        data.setNextNid(nid);
        return data;
    }
View Full Code Here

        public void run() {
            while (true) {
                try {
                    // 注意single时需传递同一个eventData对象,里面带着对应的nextNid信息
                    final EtlEventData eventData = arbitrateEventService.selectEvent().await(pipelineId);
                    // 提交对应的任务工作任务
                    executor.submit(new Callable() {

                        @Override
                        public Object call() throws Exception {
                            Long nextNid = eventData.getNextNid();
                            // 业务处理
                            Thread.sleep(500 + RandomUtils.nextInt(2000)); // sleep一下代表处理业务

                            if (isLocal(nextNid)) {// 判断是否本地
                                // 调用本地的pipe工具进行数据处理
                                eventData.setDesc(new Object());// 并设置相关信息
                            } else {
                                // 调用HTTP的pipe工具进行数据处理
                                eventData.setDesc(new Object());// 并设置相关信息
                            }

                            // 处理完成,通知一下
                            arbitrateEventService.selectEvent().single(eventData);
                            return true;
View Full Code Here

        public void run() {
            while (true) {
                try {
                    // 注意single时需传递同一个eventData对象,里面带着对应的nextNid信息
                    final EtlEventData eventData = arbitrateEventService.transformEvent().await(pipelineId);
                    // 提交对应的任务工作任务
                    executor.submit(new Callable() {

                        @Override
                        public Object call() throws Exception {
                            Long nextNid = eventData.getNextNid();
                            // 业务处理
                            Thread.sleep(500 + RandomUtils.nextInt(2000)); // sleep一下代表处理业务

                            if (isLocal(nextNid)) {// 判断是否本地
                                // 调用本地的pipe工具进行数据处理
                                eventData.setDesc(new Object());// 并设置相关信息
                            } else {
                                // 调用HTTP的pipe工具进行数据处理
                                eventData.setDesc(new Object());// 并设置相关信息
                            }

                            // 处理完成,通知一下
                            arbitrateEventService.transformEvent().single(eventData);
                            return true;
View Full Code Here

        final List<Long> initProcessIds = new ArrayList<Long>();
        try {
            // 获取数据

            // select stage
            EtlEventData sdata1 = selectEvent.await(pipelineId);
            EtlEventData sdata2 = selectEvent.await(pipelineId);

            initProcessIds.add(sdata1.getProcessId());
            initProcessIds.add(sdata2.getProcessId());

            selectEvent.single(sdata1);
            selectEvent.single(sdata2);
            // extract stage
            EtlEventData edata1 = extractEvent.await(pipelineId);
            EtlEventData edata2 = extractEvent.await(pipelineId);

            extractEvent.single(edata1);
            extractEvent.single(edata2);

            // transform stage
            EtlEventData tdata1 = transformEvent.await(pipelineId);
            EtlEventData tdata2 = transformEvent.await(pipelineId);

            transformEvent.single(tdata1);
            transformEvent.single(tdata2);

            // load stage
            EtlEventData ldata1 = loadEvent.await(pipelineId);
            loadEvent.single(ldata1);
            Long p1 = ldata1.getProcessId();

            TerminEventData terminData1 = new TerminEventData();
            terminData1.setPipelineId(pipelineId);
            terminData1.setProcessId(p1);
            terminEvent.ack(terminData1);// 发送ack信号,删除termin节点

            EtlEventData ldata2 = loadEvent.await(pipelineId);
            want.bool(ldata1.getProcessId() < ldata2.getProcessId()).is(true);
            loadEvent.single(ldata2);

            Long p2 = ldata2.getProcessId();
            TerminEventData terminData2 = new TerminEventData();
            terminData2.setPipelineId(pipelineId);
            terminData2.setProcessId(p2);
            terminEvent.ack(terminData2);// 发送ack信号,删除termin节点
View Full Code Here

        final List<Long> initProcessIds = new ArrayList<Long>();
        try {
            // 获取数据

            // select stage
            EtlEventData sdata1 = selectEvent.await(pipelineId);
            EtlEventData sdata2 = selectEvent.await(pipelineId);

            initProcessIds.add(sdata1.getProcessId());
            initProcessIds.add(sdata2.getProcessId());

            selectEvent.single(sdata1);
            selectEvent.single(sdata2);
            // extract stage
            EtlEventData edata1 = extractEvent.await(pipelineId);
            EtlEventData edata2 = extractEvent.await(pipelineId);

            extractEvent.single(edata1);
            extractEvent.single(edata2);

            // transform stage
            EtlEventData tdata1 = transformEvent.await(pipelineId);
            EtlEventData tdata2 = transformEvent.await(pipelineId);

            transformEvent.single(tdata1);
            transformEvent.single(tdata2);

            // load stage
            EtlEventData ldata1 = loadEvent.await(pipelineId);
            loadEvent.single(ldata1);
            Long p1 = ldata1.getProcessId();

            TerminEventData terminData1 = new TerminEventData();
            terminData1.setPipelineId(pipelineId);
            terminData1.setProcessId(p1);
            terminEvent.ack(terminData1);// 发送ack信号,删除termin节点

            EtlEventData ldata2 = loadEvent.await(pipelineId);
            want.bool(ldata1.getProcessId() < ldata2.getProcessId()).is(true);
            loadEvent.single(ldata2);

            Long p2 = ldata2.getProcessId();
            TerminEventData terminData2 = new TerminEventData();
            terminData2.setPipelineId(pipelineId);
            terminData2.setProcessId(p2);
            terminEvent.ack(terminData2);// 发送ack信号,删除termin节点
View Full Code Here

        public void run() {
            while (true) {
                try {
                    // 注意single时需传递同一个eventData对象,里面带着对应的nextNid信息
                    final EtlEventData eventData = arbitrateEventService.extractEvent().await(pipelineId);
                    // 提交对应的任务工作任务
                    executor.submit(new Callable() {

                        @Override
                        public Object call() throws Exception {
                            Long nextNid = eventData.getNextNid();
                            // 业务处理
                            Thread.sleep(500 + RandomUtils.nextInt(2000)); // sleep一下代表处理业务

                            if (isLocal(nextNid)) {// 判断是否本地
                                // 调用本地的pipe工具进行数据处理
                                eventData.setDesc(new Object());// 并设置相关信息
                            } else {
                                // 调用HTTP的pipe工具进行数据处理
                                eventData.setDesc(new Object());// 并设置相关信息
                            }

                            // 处理完成,通知一下
                            arbitrateEventService.extractEvent().single(eventData);
                            return true;
View Full Code Here

        final List<Long> initProcessIds = new ArrayList<Long>();
        try {
            // 获取数据

            // select stage
            EtlEventData sdata1 = selectEvent.await(pipelineId);
            EtlEventData sdata2 = selectEvent.await(pipelineId);

            initProcessIds.add(sdata1.getProcessId());
            initProcessIds.add(sdata2.getProcessId());

            selectEvent.single(sdata1);
            selectEvent.single(sdata2);
            // extract stage
            EtlEventData edata1 = extractEvent.await(pipelineId);
            EtlEventData edata2 = extractEvent.await(pipelineId);

            extractEvent.single(edata1);
            extractEvent.single(edata2);

            // transform stage
            EtlEventData tdata1 = transformEvent.await(pipelineId);
            EtlEventData tdata2 = transformEvent.await(pipelineId);

            transformEvent.single(tdata1);
            transformEvent.single(tdata2);

            SelectStageListener selectStageListener = ArbitrateFactory.getInstance(pipelineId,
                                                                                   SelectStageListener.class);
            selectStageListener.destory();
            // load stage
            EtlEventData ldata1 = loadEvent.await(pipelineId);
            loadEvent.single(ldata1);
            Long p1 = ldata1.getProcessId();

            TerminEventData terminData1 = new TerminEventData();
            terminData1.setPipelineId(pipelineId);
            terminData1.setProcessId(p1);
            terminEvent.ack(terminData1);// 发送ack信号,删除termin节点

            EtlEventData ldata2 = loadEvent.await(pipelineId);
            want.bool(ldata1.getProcessId() < ldata2.getProcessId()).is(true);
            loadEvent.single(ldata2);

            Long p2 = ldata2.getProcessId();
            TerminEventData terminData2 = new TerminEventData();
            terminData2.setPipelineId(pipelineId);
            terminData2.setProcessId(p2);
            terminEvent.ack(terminData2);// 发送ack信号,删除termin节点
View Full Code Here

        final List<Long> initProcessIds = new ArrayList<Long>();
        try {
            //获取数据

            //select stage
            EtlEventData sdata1 = selectEvent.await(pipelineId);
            EtlEventData sdata2 = selectEvent.await(pipelineId);

            initProcessIds.add(sdata1.getProcessId());
            initProcessIds.add(sdata2.getProcessId());

            selectEvent.single(sdata1);
            selectEvent.single(sdata2);
            // extract stage
            EtlEventData edata1 = extractEvent.await(pipelineId);
            EtlEventData edata2 = extractEvent.await(pipelineId);

            extractEvent.single(edata1);
            extractEvent.single(edata2);
            ArbitrateFactory.destory(pipelineId);
        } catch (InterruptedException e) {
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.arbitrate.model.EtlEventData

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.