Package com.alibaba.otter.shared.common.model.config.data

Examples of com.alibaba.otter.shared.common.model.config.data.DataMedia


            boolean flag = mustDb
                           || (eventData.getSyncConsistency() != null && eventData.getSyncConsistency().isMedia());

            // 增加一种case, 针对oracle erosa有时侯结果记录只有主键,没有变更字段,需要做一次反查
            if (!flag && CollectionUtils.isEmpty(eventData.getUpdatedColumns())) {
                DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
                if (dataMedia.getSource().getType().isOracle()) {
                    flag |= true;
                    eventData.setRemedy(true);// 针对这类数据,也统一视为补救的操作,可能erosa解析时反查数据库也不存在记录
                }
            }
View Full Code Here


    }

    private boolean checkNeedDbForRowMode(Pipeline pipeline, EventData eventData) {
        // 获取数据表信息
        DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
        DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
        Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
        if (table.getColumnCount() == eventData.getColumns().size() + eventData.getKeys().size()) {
            return false;
        } else {
            return true;
View Full Code Here

            if (StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemDualTable(), tableName)) {
                // 心跳表数据直接忽略
                return null;
            }
        } else {
            DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, schemaName, tableName);
            if (dataMedia == null) {
                throw new SelectException("the namespace = " + schemaName + " name = " + tableName
                                          + " dataMedia is null,please check , entry: " + entry.toString()
                                          + " and rowdata: " + rowChange.toString());
            }
            if (eventType.isDdl()) {
                // 如果EventType是CREATE/ALTER,需要reload DataMediaInfo;并且把CREATE/ALTER类型的事件丢弃掉.
                if (eventType.isCreate() || eventType.isAlter() || eventType.isRename()) {
                    DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                                                                        (DbMediaSource) dataMedia.getSource());
                    dbDialect.reloadTable(schemaName, tableName);// 更新下meta信息
                }

                boolean ddlSync = pipeline.getParameters().getDdlSync();
                if (ddlSync) {
                    // 处理下ddl操作
                    EventData eventData = new EventData();
                    eventData.setSchemaName(schemaName);
                    eventData.setTableName(tableName);
                    eventData.setEventType(eventType);
                    eventData.setExecuteTime(entry.getHeader().getExecuteTime());
                    eventData.setSql(rowChange.getSql());
                    eventData.setDdlSchemaName(rowChange.getDdlSchemaName());
                    eventData.setTableId(dataMedia.getId());
                    return Arrays.asList(eventData);
                }
            }
        }
View Full Code Here

        boolean useTableTransform = pipeline.getParameters().getUseTableTransform();
        EventType eventType = eventData.getEventType();
        Table table = null;
        TableInfoHolder tableHolder = null;
        DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, eventData.getSchemaName(),
                                                               eventData.getTableName());
        if (dataMedia == null) {
            throw new SelectException("the namespace = " + eventData.getSchemaName() + " name = "
                                      + eventData.getTableName() + " dataMedia is null,please check , entry: "
                                      + entry.toString() + " and rowdata: " + rowChange.toString());
        }
        eventData.setTableId(dataMedia.getId());
        if (useTableTransform || dataMedia.getSource().getType().isOracle()) {// oracle需要反查一次meta
            // 如果设置了需要进行table meta转化,则反查一下table信息
            // 比如oracle erosa解析时可能使用了非物理主键,需要直接使用,信任erosa的信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
            table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());// 查询一下meta信息
            if (table == null) {
                logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(),
                            eventData.getTableName());
            }
View Full Code Here

     *
     * @param dataMediaDo
     * @return DataMedia
     */
    private DataMedia doToModel(DataMediaDO dataMediaDo) {
        DataMedia dataMedia = null;
        try {
            DataMediaSource dataMediaSource = dataMediaSourceService.findById(dataMediaDo.getDataMediaSourceId());
            if (dataMediaSource.getType().isMysql() || dataMediaSource.getType().isOracle()) {
                dataMedia = JsonUtils.unmarshalFromString(dataMediaDo.getProperties(), DbDataMedia.class);
                dataMedia.setSource(dataMediaSource);
            } else if (dataMediaSource.getType().isNapoli() || dataMediaSource.getType().isMq()) {
                dataMedia = JsonUtils.unmarshalFromString(dataMediaDo.getProperties(), MqDataMedia.class);
                dataMedia.setSource(dataMediaSource);
            }

            dataMedia.setId(dataMediaDo.getId());
            dataMedia.setGmtCreate(dataMediaDo.getGmtCreate());
            dataMedia.setGmtModified(dataMediaDo.getGmtModified());

        } catch (Exception e) {
            logger.error("ERROR ## change the dataMedia Do to Model has an exception");
            throw new ManagerException(e);
        }
View Full Code Here

     * @param context
     * @param eventDatas
     */
    private void doDdl(DbLoadContext context, List<EventData> eventDatas) {
        for (final EventData data : eventDatas) {
            DataMedia dataMedia = ConfigHelper.findDataMedia(context.getPipeline(), data.getTableId());
            final DbDialect dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
                                                                      (DbMediaSource) dataMedia.getSource());
            Boolean skipDdlException = context.getPipeline().getParameters().getSkipDdlException();
            try {
                Boolean result = dbDialect.getJdbcTemplate().execute(new StatementCallback<Boolean>() {

                    public Boolean doInStatement(Statement stmt) throws SQLException, DataAccessException {
View Full Code Here

            this.context = context;
            this.datas = datas;
            this.canBatch = canBatch;

            EventData data = datas.get(0); // eventData为同一数据库的记录,只取第一条即可
            DataMedia dataMedia = ConfigHelper.findDataMedia(context.getPipeline(), data.getTableId());
            dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
                                                      (DbMediaSource) dataMedia.getSource());

        }
View Full Code Here

    private LoadInterceptor[] getIntercetptor(DbLoadContext context, List<EventData> currentData) {
        if (currentData == null || currentData.size() == 0) {
            return empty;
        }
        DataMedia dataMedia = ConfigHelper.findDataMedia(context.getPipeline(), currentData.get(0).getTableId());
        DbDialect dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
                                                            (DbMediaSource) dataMedia.getSource());

        if (dbDialect instanceof MysqlDialect) {
            return mysqlInterceptors;
        } else if (dbDialect instanceof OracleDialect) {
            return oracleInterceptors;
View Full Code Here

    private DbDialectFactory dbDialectFactory;

    public EventData transform(EventData data, OtterTransformerContext context) {
        EventData result = new EventData();
        // 处理Table转化
        DataMedia dataMedia = context.getDataMediaPair().getTarget();
        result.setPairId(context.getDataMediaPair().getId());
        result.setTableId(dataMedia.getId());
        // 需要特殊处理下multi场景
        buildName(data, result, context.getDataMediaPair());
        result.setEventType(data.getEventType());
        result.setExecuteTime(data.getExecuteTime());
        result.setSyncConsistency(data.getSyncConsistency());
        result.setRemedy(data.isRemedy());
        result.setSyncMode(data.getSyncMode());
        result.setSize(data.getSize());
        if (data.getEventType().isDdl()) {
            // ddl不需要处理字段
            if (StringUtils.equalsIgnoreCase(result.getSchemaName(), data.getSchemaName())
                && StringUtils.equalsIgnoreCase(result.getTableName(), data.getTableName())) {
                // 是否需要对ddl sql进行转化,暂时不支持异构,必须保证源表和目标表的名字相同
                result.setDdlSchemaName(data.getDdlSchemaName());
                result.setSql(data.getSql());
                return result;
            } else {
                throw new TransformException("no support ddl for [" + data.getSchemaName() + "." + data.getTableName()
                                             + "] to [" + result.getSchemaName() + "." + result.getTableName()
                                             + "] , sql :" + data.getSql());
            }
        }

        Multimap<String, String> translateColumnNames = HashMultimap.create();
        if (context.getDataMediaPair().getColumnPairMode().isInclude()) { // 只针对正向匹配进行名字映射,exclude不做处理
            List<ColumnPair> columnPairs = context.getDataMediaPair().getColumnPairs();
            for (ColumnPair columnPair : columnPairs) {
                translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
            }
        }
        // 准备一下table meta
        DataMediaPair dataMediaPair = context.getDataMediaPair();
        boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
        boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
        TableInfoHolder tableHolder = null;
        if (useTableTransform || enableCompatibleMissColumn) {// 控制一下是否需要反查table meta信息,如果同构数据库,完全没必要反查
            // 获取目标库的表信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(),
                                                                (DbMediaSource) dataMedia.getSource());

            Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
            tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
        }
View Full Code Here

     * 2. 源:offer[1-128] , 目:offer
     * 3. 源:offer[1-128] , 目:offer[1-128]
     * 4. 源:offer , 目:offer[1-128] 不支持,会报错
     */
    private void buildName(EventData data, EventData result, DataMediaPair pair) {
        DataMedia targetDataMedia = pair.getTarget();
        DataMedia sourceDataMedia = pair.getSource();
        String schemaName = buildName(data.getSchemaName(), sourceDataMedia.getNamespaceMode(),
                                      targetDataMedia.getNamespaceMode());
        String tableName = buildName(data.getTableName(), sourceDataMedia.getNameMode(), targetDataMedia.getNameMode());
        result.setSchemaName(schemaName);
        result.setTableName(tableName);
    }
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.common.model.config.data.DataMedia

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.