Examples of ImportTargetTableBean


Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    @Test
    public void sendImportFileTtest01() throws Exception {
        // ImportBeanを生成
        File importFile = new File("src/test/data/importer/IMP_IMPORT_TARGET1.tsv");
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean = new ImportTargetTableBean();
        tableBean.setImportProtocol(FileList.content("dummy1"));
        tableBean.setImportFile(importFile);
        targetTable.put("IMPORT_TARGET1", tableBean);
        ImportBean bean = new ImportBean();
        bean.setTargetTable(targetTable);
        bean.setJobflowId("11");
        bean.setExecutionId("11-11");
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    public void sendImportFileTtest02() throws Exception {
        // ImportBeanを生成
        File importFile1 = new File("src/test/data/importer/IMP_IMPORT_TARGET1.tsv");
        File importFile2 = new File("src/test/data/importer/IMP_IMPORT_TARGET2.tsv");
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setImportProtocol(FileList.content("dummy1"));
        tableBean1.setImportFile(importFile1);
        targetTable.put("IMPORT_TARGET1", tableBean1);
        ImportTargetTableBean tableBean2 = new ImportTargetTableBean();
        tableBean2.setImportProtocol(FileList.content("dummy2"));
        tableBean2.setImportFile(importFile2);
        targetTable.put("IMPORT_TARGET2", tableBean2);
        ImportBean bean = new ImportBean();
        bean.setTargetTable(targetTable);
        bean.setJobflowId("22");
        bean.setExecutionId("22-22");
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    @Test
    public void sendImportFileTtest03() throws Exception {
        // ImportBeanを生成
        File importFile = new File("src/test/data/importer/IMP_IMPORT_TARGET2.tsv");
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean = new ImportTargetTableBean();
        tableBean.setImportProtocol(FileList.content("dummy1"));
        tableBean.setImportFile(importFile);
        targetTable.put("IMPORT_TARGET1", tableBean);
        ImportBean bean = new ImportBean();
        bean.setTargetTable(targetTable);
        bean.setJobflowId("11");
        bean.setExecutionId("11-11");
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    @Test
    public void sendImportFileTtest04() throws Exception {
        // ImportBeanを生成
        File importFile = new File("src/test/data/importer/IMP_IMPORT_TARGET99.tsv");
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean = new ImportTargetTableBean();
        tableBean.setImportProtocol(FileList.content("dummy1"));
        tableBean.setImportFile(importFile);
        targetTable.put("IMPORT_TARGET1", tableBean);
        ImportBean bean = new ImportBean();
        bean.setTargetTable(targetTable);
        bean.setJobflowId("11");
        bean.setExecutionId("11-11");
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    @Test
    public void sendImportFileTtest05() throws Exception {
        // ImportBeanを生成
        File importFile = new File("src/test/data/importer/IMP_IMPORT_TARGET1.tsv");
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean = new ImportTargetTableBean();
        tableBean.setImportProtocol(FileList.content("dummy1"));
        tableBean.setImportFile(importFile);
        targetTable.put("IMPORT_TARGET1", tableBean);
        ImportBean bean = new ImportBean();
        bean.setTargetTable(targetTable);
        bean.setJobflowId("11");
        bean.setExecutionId("11-11");
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

        }
        if (importTargetTables.isEmpty()) {
            return true;
        }
        for (Map.Entry<String, ImportTargetTableBean> entry : importTargetTables.entrySet()) {
            ImportTargetTableBean tableInfo = entry.getValue();
            tableInfo.setSearchCondition(null);
        }
        boolean result = checkImportParam(importTargetTables, targetName, jobflowId, propFile.getPath(), true);
        return result;
    }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

            return false;
        }
        Map<String, ImportTargetTableBean> cacheTables = new HashMap<String, ImportTargetTableBean>();
        for (Map.Entry<String, ImportTargetTableBean> entry : importTargetTables.entrySet()) {
            String tableName = entry.getKey();
            ImportTargetTableBean tableInfo = entry.getValue();
            if (tableInfo.getCacheId() == null) {
                LOG.debugMessage("Table \"{0}\" does not use cache mechanism", tableName);
                continue;
            }
            tableInfo.setLockType(ImportTableLockType.NONE);
            tableInfo.setLockedOperation(ImportTableLockedOperation.FORCE);
            cacheTables.put(tableName, tableInfo);
        }
        if (cacheTables.isEmpty()) {
            return true;
        }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

        }
        List<String> targetTable = spritComma(strTargetTable);

        // Import対象テーブル分のImportTargetTableBeanを作成する
        for (String element : targetTable) {
            importTargetTables.put(element, new ImportTargetTableBean());
        }

        // ImportTargetTableBeanに値を詰める
        Set<?> set = importProp.keySet();
        for (Object objKey : set) {
            String key = (String) objKey;
            // KEYが対象テーブルの設定でない場合にその行を解析する
            if (!IMP_TARGET_TABLE.equals(key)) {

                // TODO 一連の文字列を解析する処理の見通しが悪い
                // テーブル名を取得
                String tableName = key.substring(0, key.indexOf('.'));

                // 当該行がテーブルに対する何の設定か取得
                int start = tableName.length() + 3;
                int end = key.indexOf('=', start) - 1;
                String keyMeans = key.substring(start + end);

                // テーブル名がImport対象テーブルとして定義されていない場合は読み飛ばす
                ImportTargetTableBean bean = importTargetTables.get(tableName);
                if (bean == null) {
                    continue;
                }
                String value = importProp.getProperty(key);
                // KEYに対するVALUEがない場合は読み飛ばす
                if (value == null || value.equals("")) {
                    continue;
                }

                // 当該行の設定をImportTargetTableBeanに設定する
                if (IMP_TARGET_COLUMN.equals(keyMeans)) {
                    // Import対象カラム
                    bean.setImportTargetColumns(spritComma(value));
                } else if (IMP_SEARCH_CONDITION.equals(keyMeans)) {
                    bean.setSearchCondition(value);
                } else if (IMP_CACHE_ID.equals(keyMeans)) {
                    // enables cache
                    if (value.trim().isEmpty() == false) {
                        bean.setCacheId(value);
                    }
                } else if (IMP_LOCK_TYPE.equals(keyMeans)) {
                    // ロック取得タイプ
                    bean.setLockType(ImportTableLockType.find(value));
                } else if (IMP_LOCKED_OPERATION.equals(keyMeans)) {
                    // ロック済みの場合の取り扱い
                    bean.setLockedOperation(ImportTableLockedOperation.find(value));
                } else if (IMP_BEAN_NAME.equals(keyMeans)) {
                    // JavaBeansクラス名
                    try {
                        bean.setImportTargetType(loadClass(value));
                    } catch (ClassNotFoundException e) {
                        LOG.error(e, "TG-COMMON-00002",
                                "Import対象テーブルに対応するJavaBeanのクラスが存在しない",
                                targetName, jobflowId, propFilePath);
                        return false;
                    }
                } else if (IMP_HDFS_IMPORT_FILE.equals(keyMeans)) {
                    // HDFS上の出力パス
                    bean.setDfsFilePath(value);
                } else {
                    // 設定が不明の場合は読み飛ばす
                    LOG.warn("TG-COMMON-00006",
                            "不明な設定。 key:" + key,
                            targetName, jobflowId, tableName, propFilePath);
                    continue;
                }
            }
        }
        for (Map.Entry<String, ImportTargetTableBean> entry : importTargetTables.entrySet()) {
            ImportTargetTableBean bean = entry.getValue();
            if (bean.getCacheId() != null) {
                if (ThunderGateCacheSupport.class.isAssignableFrom(bean.getImportTargetType()) == false) {
                    LOG.error("TG-COMMON-00002",
                            MessageFormat.format(
                                    "データモデルクラス\"{1}\"がキャッシュをサポートしていない ({0})",
                                    ThunderGateCacheSupport.class.getName(),
                                    bean.getImportTargetType().getName()),
                            targetName, jobflowId, propFilePath);
                    return false;
                }
                if (bean.getSearchCondition() != null && bean.getSearchCondition().trim().isEmpty() == false) {
                    LOG.error("TG-COMMON-00002",
                            MessageFormat.format(
                                    "キャッシュ利用時に条件式を指定している ({0})",
                                    bean.getLockedOperation()),
                            targetName, jobflowId, propFilePath);
                    return false;
                }
                if (bean.getLockedOperation() == ImportTableLockedOperation.OFF) {
                    LOG.error("TG-COMMON-00002",
                            MessageFormat.format(
                                    "キャッシュ利用時にロック箇所を読み飛ばす設定がされている ({0})",
                                    bean.getLockedOperation()),
                            targetName, jobflowId, propFilePath);
                    return false;
                }
            }
        }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

            String jobflowId,
            String fileName,
            boolean isPrimary) {
        for (Map.Entry<String, ImportTargetTableBean> entry : tables.entrySet()) {
            String tableName = entry.getKey();
            ImportTargetTableBean bean = entry.getValue();

            // Import対象カラムのチェック
            if (isEmptyOrHasEmptyString(bean.getImportTargetColumns())) {
                LOG.error("TG-COMMON-00004",
                        "Import対象カラムが設定されていない",
                        targetName, jobflowId, tableName, fileName);
                return false;
            }

            // ロック取得のタイプのチェック
            ImportTableLockType lockType = bean.getLockType();
            if (lockType == null) {
                LOG.error("TG-COMMON-00004",
                        "ロック取得タイプの設定が不正",
                        targetName, jobflowId, tableName, fileName);
                return false;
            } else {
                if (!isPrimary && !ImportTableLockType.NONE.equals(lockType)) {
                    LOG.error("TG-COMMON-00004",
                            "サブ起動のImporterに対してロック取得タイプ「3:ロックしない」以外が指定されている。",
                            targetName, jobflowId, tableName, fileName);
                    return false;
                }
            }

            // ロック済みの場合の取り扱いのチェック
            ImportTableLockedOperation operation = bean.getLockedOperation();
            if (operation == null) {
                LOG.error("TG-COMMON-00004",
                        "ロック済みの場合の取り扱いの設定が不正",
                        targetName, jobflowId, tableName, fileName);
                return false;
            } else {
                if (!isPrimary && !ImportTableLockedOperation.FORCE.equals(operation)) {
                    LOG.error("TG-COMMON-00004",
                            "サブ起動のImporterに対してロック済みの場合の取り扱い「2:ロックの有無にかかわらず処理対象とする」以外が指定されている。",
                            targetName, jobflowId, tableName, fileName);
                    return false;
                }
            }

            // ロック取得のタイプとロック済みの場合の取り扱いの組合せのチェック
            if (lockType.equals(ImportTableLockType.TABLE) && operation.equals(ImportTableLockedOperation.OFF)
                    || lockType.equals(ImportTableLockType.TABLE) && operation.equals(ImportTableLockedOperation.FORCE)
                    || lockType.equals(ImportTableLockType.RECORD) && operation.equals(ImportTableLockedOperation.FORCE)
                    || lockType.equals(ImportTableLockType.NONE) && operation.equals(ImportTableLockedOperation.OFF)) {
                LOG.error("TG-COMMON-00004",
                        "ロック取得のタイプとロック済みの場合の取り扱いの組合せが不正",
                        targetName, jobflowId, tableName, fileName);
                return false;
            }

            // JavaBeansのクラス名のチェック
            Class<?> beanClass = bean.getImportTargetType();
            if (beanClass == null) {
                LOG.error("TG-COMMON-00004",
                        "Import対象テーブルに対応するJavaBeanのクラスが未設定",
                        targetName, jobflowId, tableName, fileName);
                return false;
            }

            // HDFS上のファイルパスをチェック
            String path = bean.getDfsFilePath();
            if (isEmpty(path)) {
                LOG.error("TG-COMMON-00004",
                        "Import対象テーブルのデータをHDFS上に書き出す際のファイルパスが未設定",
                        targetName, jobflowId, tableName, fileName);
                return false;
            } else {
                try {
                    // 有効なパスかチェック
                    VariableTable variables = Constants.createVariableTable();
                    variables.defineVariable(Constants.HDFS_PATH_VARIABLE_USER, "dummyuser");
                    variables.defineVariable(Constants.HDFS_PATH_VARIABLE_EXECUTION_ID, "dummyid");
                    String dummyPath = variables.parse(path, false);
                    new URI(dummyPath).normalize();
                } catch (URISyntaxException e) {
                    LOG.error(e, "TG-COMMON-00004",
                            "HDFS上に書き出す際のファイルパスが有効でない",
                            targetName, jobflowId, tableName, fileName);
                    return false;
                }
            }

            // 検索条件(検索条件はcreateSerchConditionを使って置換を行う)
            String condition = bean.getSearchCondition();
            if (condition != null) {
                condition = createSearchCondition(condition, targetName, jobflowId, fileName);
                if (condition == null) {
                    return false;
                } else {
                    bean.setSearchCondition(condition);
                }
            }
        }
        return true;
    }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

        }

        // Import対象テーブル毎に処理し、指定に応じてロックフラグを立てる
        List<String> list = bean.getImportTargetTableList();
        for (String tableName : list) {
            ImportTargetTableBean targetTable = bean.getTargetTable(tableName);
            ImportTableLockType lockType = targetTable.getLockType();
            ImportTableLockedOperation operation = targetTable.getLockedOperation();
            String serchCondition = targetTable.getSearchCondition();

            LOG.info("TG-IMPORTER-02009",
                    tableName, lockType, operation, serchCondition);

            // 「ロックしない」かつ「ロック有無に関わらず処理対象とする」の場合、ロックフラグは立てずに終了する。
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.