Package com.alibaba.otter.node.etl.common.db.dialect

Examples of com.alibaba.otter.node.etl.common.db.dialect.DbDialect


    }

    private boolean checkNeedDbForRowMode(Pipeline pipeline, EventData eventData) {
        // 获取数据表信息
        DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
        DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
        Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
        if (table.getColumnCount() == eventData.getColumns().size() + eventData.getKeys().size()) {
            return false;
        } else {
            return true;
        }
View Full Code Here


            try {
                MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
                Thread.currentThread().setName(String.format(WORKER_NAME_FORMAT, pipeline.getId(), pipeline.getName()));
                // 获取数据表信息
                DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
                DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                                                                    (DbMediaSource) dataMedia.getSource());
                Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                TableData keyTableData = buildTableData(table, eventData.getKeys());

                // oracle类型特殊处理下
                if (dbDialect instanceof OracleDialect) {
                    keyTableData.columnTypes = getOraclePkTypes(table, keyTableData.columnNames);
View Full Code Here

        boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
        TableInfoHolder tableHolder = null;
        if (useTableTransform || enableCompatibleMissColumn) {// 控制一下是否需要反查table
                                                              // meta信息,如果同构数据库,完全没必要反查
            // 获取目标库的表信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(),
                (DbMediaSource) dataMedia.getSource());

            Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
            tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
        }

        // 处理column转化
        List<EventColumn> otherColumns = translateColumns(result,
View Full Code Here

            Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
            // 匹配字段为空,可能源库发生过DDL操作,目标库重新载入一下meta信息
            if (matchDbColumn == null) { // 尝试reload一下table meta
                // 获取目标库的表信息
                DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource();
                DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource);
                String schemaName = tableHolder.getTable().getSchema();
                if (StringUtils.isEmpty(schemaName)) {
                    schemaName = tableHolder.getTable().getCatalog();
                }
                Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false); // 强制反查一次,并放入cache

                tableHolder.setTable(table);
                matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
                if (matchDbColumn == null) {
                    if (canColumnsNotExist) {
View Full Code Here

    private DbDialectFactory dbDialectFactory;

    public boolean before(DbLoadContext context, EventData currentData) {
        // 初步构建sql
        DbDialect dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
            (DbMediaSource) context.getDataMediaSource());
        SqlTemplate sqlTemplate = dbDialect.getSqlTemplate();
        EventType type = currentData.getEventType();
        String sql = null;

        String schemaName = (currentData.isWithoutSchema() ? null : currentData.getSchemaName());
        // 注意insert/update语句对应的字段数序都是将主键排在后面
View Full Code Here

                            // 如果指定了tableId,需要按照tableId进行严格查找,如果没找到,那说明不需要进行同步
                            dataMedia = ConfigHelper.findDataMedia(pipeline,
                                Long.valueOf(tableIdColumn.getColumnValue()));
                        }

                        DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                            (DbMediaSource) dataMedia.getSource());
                        // 考虑offer[1-128]的配置模式
                        if (!dataMedia.getNameMode().getMode().isSingle()
                            || !dataMedia.getNamespaceMode().getMode().isSingle()) {
                            boolean hasError = true;
                            EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                            if (fullNameColumn != null) {
                                String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                                if (names.length >= 2) {
                                    eventData.setSchemaName(names[0]);
                                    eventData.setTableName(names[1]);
                                    hasError = false;
                                }
                            }

                            if (hasError) {
                                // 出现异常,需要记录一下
                                logger.warn("dataMedia mode:{} , fullname:{} ",
                                    dataMedia.getMode(),
                                    fullNameColumn == null ? null : fullNameColumn.getColumnValue());
                                removeDatas.add(eventData);
                                // 跳过这条记录
                                continue;
                            }
                        } else {
                            eventData.setSchemaName(dataMedia.getNamespace());
                            eventData.setTableName(dataMedia.getName());
                        }

                        // 更新业务类型
                        EventColumn typeColumn = getMatchColumn(eventData.getColumns(), TYPE);
                        EventType eventType = EventType.valuesOf(typeColumn.getColumnValue());
                        eventData.setEventType(eventType);
                        if (eventType.isUpdate()) {// 如果是update强制修改为insert,这样可以在目标端执行merge
                                                   // sql
                            eventData.setEventType(EventType.INSERT);
                        } else if (eventType.isDdl()) {
                            dbDialect.reloadTable(eventData.getSchemaName(), eventData.getTableName());
                            removeDatas.add(eventData);// 删除当前记录
                            continue;
                        }
                        // 重新构建新的业务主键字段
                        EventColumn pkDataColumn = getMatchColumn(eventData.getColumns(), PK_DATA);
                        String pkData = pkDataColumn.getColumnValue();
                        String[] pks = StringUtils.split(pkData, PK_SPLIT);

                        Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                        List<EventColumn> newColumns = new ArrayList<EventColumn>();
                        Column[] primaryKeyColumns = table.getPrimaryKeyColumns();
                        if (primaryKeyColumns.length > pks.length) {
                            throw new ExtractException("data pk column size not match , data:" + eventData.toString());
                        }
View Full Code Here

            if (eventType.isDdl()) {
                DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, schemaName, tableName);
                // 如果EventType是CREATE/ALTER,需要reload
                // DataMediaInfo;并且把CREATE/ALTER类型的事件丢弃掉.
                if (eventType.isCreate() || eventType.isAlter() || eventType.isRename()) {
                    DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                        (DbMediaSource) dataMedia.getSource());
                    dbDialect.reloadTable(schemaName, tableName);// 更新下meta信息
                }

                boolean ddlSync = pipeline.getParameters().getDdlSync();
                if (ddlSync) {
                    // 处理下ddl操作
View Full Code Here

                eventData.getTableName());
            eventData.setTableId(dataMedia.getId());
            if (useTableTransform || dataMedia.getSource().getType().isOracle()) {// oracle需要反查一次meta
                // 如果设置了需要进行table meta转化,则反查一下table信息
                // 比如oracle erosa解析时可能使用了非物理主键,需要直接使用,信任erosa的信息
                DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                    (DbMediaSource) dataMedia.getSource());
                table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());// 查询一下meta信息
                if (table == null) {
                    logger.warn("find table[{}.{}] is null , may be drop table.",
                        eventData.getSchemaName(),
                        eventData.getTableName());
                }
View Full Code Here

    }

    private boolean checkNeedDbForRowMode(Pipeline pipeline, EventData eventData) {
        // 获取数据表信息
        DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
        DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
        Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
        if (table.getColumnCount() == eventData.getColumns().size() + eventData.getKeys().size()) {
            return false;
        } else {
            return true;
        }
View Full Code Here

            try {
                MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
                Thread.currentThread().setName(String.format(WORKER_NAME_FORMAT, pipeline.getId(), pipeline.getName()));
                // 获取数据表信息
                DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
                DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                    (DbMediaSource) dataMedia.getSource());
                Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                TableData keyTableData = buildTableData(table, eventData.getKeys());

                // oracle类型特殊处理下
                if (dbDialect instanceof OracleDialect) {
                    keyTableData.columnTypes = getOraclePkTypes(table, keyTableData.columnNames);
View Full Code Here

TOP

Related Classes of com.alibaba.otter.node.etl.common.db.dialect.DbDialect

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.