Examples of com.alibaba.otter.canal.protocol.position.EntryPosition

Package com.alibaba.otter.canal.protocol.position

Examples of com.alibaba.otter.canal.protocol.position.EntryPosition

com.alibaba.otter.canal.protocol.position.EntryPosition
数据库对象的唯一标示 @author jianghang 2012-6-14 下午09:20:07 @version 1.0.0

                                                                           parameters.getDefaultDatabaseName()));
                }
            }


            if (!CollectionUtils.isEmpty(parameters.getPositions())) {
                EntryPosition masterPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0),
                                                                             EntryPosition.class);
                // binlog位置参数
                mysqlEventParser.setMasterPosition(masterPosition);


                if (parameters.getPositions().size() > 1) {
                    EntryPosition standbyPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0),
                                                                                  EntryPosition.class);
                    mysqlEventParser.setStandbyPosition(standbyPosition);
                }
            }
            mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());

View Full Code Here

        connection.setSlaveId(this.slaveId);
        return connection;
    }


    protected EntryPosition findStartPosition(ErosaConnection connection) throws IOException {
        EntryPosition startPosition = findStartPositionInternal(connection);
        if (needTransactionPosition.get()) {
            logger.info("prepare to find last position : {}", startPosition.toString());
            Long preTransactionStartPosition = findTransactionBeginPosition(connection, startPosition);
            if (!preTransactionStartPosition.equals(startPosition.getPosition())) {
                logger.info("find new start Transaction Position , old : {} , new : {}", startPosition.getPosition(),
                            preTransactionStartPosition);
                startPosition.setPosition(preTransactionStartPosition);
            }
            needTransactionPosition.compareAndSet(true, false);
        }
        return startPosition;
    }

View Full Code Here


    protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
        MysqlConnection mysqlConnection = (MysqlConnection) connection;
        LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
        if (logPosition == null) {// 找不到历史成功记录
            EntryPosition entryPosition = null;
            if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
                entryPosition = masterPosition;
            } else if (standbyInfo != null
                       && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
                entryPosition = standbyPosition;
            }


            if (entryPosition == null) {
                entryPosition = findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费
            }


            // 判断一下是否需要按时间订阅
            if (StringUtils.isEmpty(entryPosition.getJournalName())) {
                // 如果没有指定binlogName，尝试按照timestamp进行查找
                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                    return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
                } else {
                    return findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费
                }
            } else {
                if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
                    // 如果指定binlogName + offest，直接返回
                    return entryPosition;
                } else {
                    EntryPosition specificLogFilePosition = null;
                    if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                        // 如果指定binlogName + timestamp，但没有指定对应的offest，尝试根据时间找一下offest
                        EntryPosition endPosition = findEndPosition(mysqlConnection);
                        if (endPosition != null) {
                            specificLogFilePosition = findAsPerTimestampInSpecificLogFile(
                                                                                          mysqlConnection,
                                                                                          entryPosition.getTimestamp(),
                                                                                          endPosition,

View Full Code Here

        }
    }


    // 根据时间查找binlog位置
    private EntryPosition findByStartTimeStamp(MysqlConnection mysqlConnection, Long startTimestamp) {
        EntryPosition endPosition = findEndPosition(mysqlConnection);
        EntryPosition startPosition = findStartPosition(mysqlConnection);
        String maxBinlogFileName = startPosition.getJournalName();
        String minBinlogFileName = startPosition.getJournalName();
        logger.info("show master status to set search end condition:{} ", endPosition);
        String startSearchBinlogFile = endPosition.getJournalName();
        boolean shouldBreak = false;
        while (running && !shouldBreak) {
            try {
                EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection, startTimestamp,
                                                                                  endPosition, startSearchBinlogFile);
                if (entryPosition == null) {
                    if (StringUtils.equalsIgnoreCase(minBinlogFileName, startSearchBinlogFile)) {
                        // 已经找到最早的一个binlog，没必要往前找了
                        shouldBreak = true;

View Full Code Here

            List<String> fields = packet.getFieldValues();
            if (CollectionUtils.isEmpty(fields)) {
                throw new CanalParseException(
                                              "command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
            }
            EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
            return endPosition;
        } catch (IOException e) {
            throw new CanalParseException("command : 'show master status' has an error!", e);
        }
    }

View Full Code Here

            List<String> fields = packet.getFieldValues();
            if (CollectionUtils.isEmpty(fields)) {
                throw new CanalParseException(
                                              "command : 'show binlog events limit 1' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
            }
            EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
            return endPosition;
        } catch (IOException e) {
            throw new CanalParseException("command : 'show binlog events limit 1' has an error!", e);
        }

View Full Code Here

            mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction<LogEvent>() {


                private LogPosition lastPosition;


                public boolean sink(LogEvent event) {
                    EntryPosition entryPosition = null;
                    try {
                        CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                        if (entry == null) {
                            return true;
                        }


                        String logfilename = entry.getHeader().getLogfileName();
                        Long logfileoffset = entry.getHeader().getLogfileOffset();
                        Long logposTimestamp = entry.getHeader().getExecuteTime();


                        if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
                            logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] {
                                    logfilename, logfileoffset, logposTimestamp, startTimestamp });
                            // 寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出
                            if (logposTimestamp >= startTimestamp) {
                                return false;
                            }
                        }


                        if (StringUtils.equals(endPosition.getJournalName(), logfilename)
                            && endPosition.getPosition() <= (logfileoffset + event.getEventLen())) {
                            return false;
                        }


                        // 记录一下上一个事务结束的位置，即下一个事务的position
                        // position = current + data.length，代表该事务的下一条offest，避免多余的事务重复
                        if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                            entryPosition = new EntryPosition(logfilename, logfileoffset + event.getEventLen(),
                                                              logposTimestamp);
                            logger.debug("set {} to be pending start position before finding another proper one...",
                                         entryPosition);
                            logPosition.setPostion(entryPosition);
                        }

View Full Code Here

                        // 开始执行replication
                        // 1. 构造Erosa连接
                        erosaConnection = buildErosaConnection();
                        erosaConnection.connect();// 链接
                        // 2. 获取最后的位置信息
                        final EntryPosition startPosition = findStartPosition(erosaConnection);
                        if (startPosition == null) {
                            throw new CanalParseException("can't find start position for " + destination);
                        }
                        logger.info("find start position : {}", startPosition.toString());
                        // 重新链接，因为在找position过程中可能有状态，需要断开后重建
                        erosaConnection.reconnect();
                        // 3. 执行dump前的准备工作
                        preDump(erosaConnection);


                        final SinkFunction sinkHandler = new SinkFunction<EVENT>() {


                            private LogPosition lastPosition;


                            public boolean sink(EVENT event) {
                                try {
                                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);


                                    if (!running) {
                                        return false;
                                    }


                                    if (entry != null) {
                                        transactionBuffer.add(entry);
                                        // 记录一下对应的positions
                                        this.lastPosition = buildLastPosition(entry);
                                    }
                                    return running;
                                } catch (TableIdNotFoundException e) {
                                    throw e;
                                } catch (Exception e) {
                                    processError(e, this.lastPosition, startPosition.getJournalName(),
                                                 startPosition.getPosition());
                                    // 走到这一步，说明出错了
                                    return false;
                                }
                            }


                        };


                        // 4. 开始dump数据
                        if (StringUtils.isEmpty(startPosition.getJournalName()) && startPosition.getTimestamp() != null) {
                            erosaConnection.dump(startPosition.getTimestamp(), sinkHandler);
                        } else {
                            erosaConnection.dump(startPosition.getJournalName(), startPosition.getPosition(),
                                                 sinkHandler);
                        }


                    } catch (TableIdNotFoundException e) {
                        // 特殊处理TableIdNotFound异常,出现这样的异常，一种可能就是起始的position是一个事务当中，导致tablemap Event时间没解析过

View Full Code Here

    protected LogPosition buildLastTranasctionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
        for (int i = entries.size() - 1; i > 0; i--) {
            CanalEntry.Entry entry = entries.get(i);
            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
                LogPosition logPosition = new LogPosition();
                EntryPosition position = new EntryPosition();
                position.setJournalName(entry.getHeader().getLogfileName());
                position.setPosition(entry.getHeader().getLogfileOffset());
                position.setTimestamp(entry.getHeader().getExecuteTime());
                logPosition.setPostion(position);


                LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
                logPosition.setIdentity(identity);
                return logPosition;

View Full Code Here

        return null;
    }


    protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
        LogPosition logPosition = new LogPosition();
        EntryPosition position = new EntryPosition();
        position.setJournalName(entry.getHeader().getLogfileName());
        position.setPosition(entry.getHeader().getLogfileOffset());
        position.setTimestamp(entry.getHeader().getExecuteTime());
        logPosition.setPostion(position);


        LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
        logPosition.setIdentity(identity);
        return logPosition;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.alibaba.otter.canal.protocol.position.EntryPosition

com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager

com.alibaba.otter.canal.meta.AbstractMetaManagerTest

com.alibaba.otter.canal.parse.inbound.AbstractEventParser

com.alibaba.otter.canal.parse.inbound.group.GroupEventPaserTest

com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogDumpTest

com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser

com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParserTest

com.alibaba.otter.canal.parse.inbound.mysql.MysqlDumpTest

com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser

com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParserTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.