Examples of ImportTargetTableBean


Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

     */
    public void deleteFile(ImportBean bean) {
        // Import対象ファイルを削除
        List<String> list = bean.getImportTargetTableList();
        for (String tableName : list) {
            ImportTargetTableBean targetTable = bean.getTargetTable(tableName);
            File file = targetTable.getImportFile();
            // ファイルが存在する場合は削除する。
            if (file != null && file.exists()) {
                if (!file.delete()) {
                    LOG.warn("TG-IMPORTER-05001", file.getPath());
                }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

        assert content != null;
        assert bean != null;
        assert user != null;
        String tableName = FileNameUtil.getImportTableName(protocol.getLocation());

        ImportTargetTableBean targetTableBean = bean.getTargetTable(tableName);
        if (targetTableBean == null) {
            // 対応するテーブルの定義がDSL存在しない場合異常終了する。
            throw new BulkLoaderSystemException(getClass(), "TG-EXTRACTOR-02001",
                    MessageFormat.format(
                            "エントリに対応するテーブルの定義がDSL存在しない。テーブル名:{0}",
                            tableName));
        }

        URI dfsFilePath = resolveLocation(bean, user, targetTableBean.getDfsFilePath());
        Class<?> targetTableModel = targetTableBean.getImportTargetType();

        LOG.info("TG-EXTRACTOR-02002",
                tableName, dfsFilePath.toString(), targetTableModel.toString());

        // ファイルをジョブ入力データ領域に書き出す
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

            || protocol.getKind() == FileProtocol.Kind.UPDATE_CACHE;

        CacheInfo info = protocol.getInfo();
        assert info != null;

        ImportTargetTableBean targetTableBean = bean.getTargetTable(info.getTableName());
        if (targetTableBean == null) {
            // 対応するテーブルの定義がDSL存在しない場合異常終了する。
            throw new BulkLoaderSystemException(getClass(), "TG-EXTRACTOR-02001",
                    MessageFormat.format(
                            "エントリに対応するテーブルの定義がDSL存在しない。テーブル名:{0}",
                            info.getTableName()));
        }

        URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation());
        try {
            CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath);
            try {
                LOG.info("TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties());
                storage.putPatchCacheInfo(info);
                LOG.info("TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties());

                Class<?> targetTableModel = targetTableBean.getImportTargetType();
                Path targetUri = storage.getPatchContents("0");
                LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri);
                long recordCount = write(targetTableModel, targetUri.toUri(), content);
                LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount);
                LOG.info("TG-PROFILE-01002",
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

        JobFlowParamLoader loader = new JobFlowParamLoader();
        if (loader.loadCacheBuildParam(targetName, batchId, jobflowId) == false) {
            return null;
        }
        ImportTargetTableBean table = null;
        for (Map.Entry<String, ImportTargetTableBean> entry : loader.getImportTargetTables().entrySet()) {
            if (entry.getKey().equals(tableName)) {
                table = entry.getValue();
                break;
            }
        }
        if (table == null) {
            LOG.error("TG-BUILDCACHE-01005", targetName, batchId, jobflowId, executionId, tableName);
            return null;
        }
        if (table.getCacheId() == null) {
            LOG.error("TG-BUILDCACHE-01006", targetName, batchId, jobflowId, executionId, tableName);
            return null;
        }
        bean.setTargetTable(Collections.singletonMap(tableName, table));
        return bean;
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

        // Import設定の検証
        // 実行結果を検証
        assertTrue(result);
        // テーブルXXXの検証
        ImportTargetTableBean impTable1 = importTargetTable.get("XXX");
        // Import対象カラム
        assertEquals(3, impTable1.getImportTargetColumns().size());
        assertEquals("columnA", impTable1.getImportTargetColumns().get(0));
        assertEquals("columnB", impTable1.getImportTargetColumns().get(1));
        assertEquals("columnC", impTable1.getImportTargetColumns().get(2));
        // 検索条件
        assertEquals("columnA='1' and columnB='2'", impTable1.getSearchCondition());
        // ロック取得タイプ
        assertEquals(ImportTableLockType.find("1"), impTable1.getLockType());
        // ロック済みの場合の取り扱い
        assertEquals(ImportTableLockedOperation.find("3"), impTable1.getLockedOperation());
        // JavaBeansクラス名
        assertEquals("com.asakusafw.bulkloader.bean.ImportTargetTableBean", impTable1.getImportTargetType().getName());
        // HDFS上の出力パス
        assertEquals("/${user}/${execution_id}/import_target1", impTable1.getDfsFilePath());

        // テーブルYYYの検証
        ImportTargetTableBean impTable2 = importTargetTable.get("YYY");
        // Import対象カラム
        assertEquals(1, impTable2.getImportTargetColumns().size());
        assertEquals("columnA", impTable2.getImportTargetColumns().get(0));
        // 検索条件
        assertEquals("columnA='1' or columnB=(select columnB from tableA where column='3')", impTable2.getSearchCondition());
        // キャッシュ利用有無
        assertEquals(false, impTable2.isUseCache());
        // ロック取得タイプ
        assertEquals(ImportTableLockType.find("2"), impTable2.getLockType());
        // ロック済みの場合の取り扱い
        assertEquals(ImportTableLockedOperation.find("1"), impTable2.getLockedOperation());
        // JavaBeansクラス名
        assertEquals("com.asakusafw.bulkloader.bean.ImportBean", impTable2.getImportTargetType().getName());
        // HDFS上の出力パス
        assertEquals("/asakusa/import/11/YYY", impTable2.getDfsFilePath());

        // テーブルZZZの検証
        ImportTargetTableBean impTable3 = importTargetTable.get("ZZZ");
        // Import対象カラム
        assertEquals(2, impTable3.getImportTargetColumns().size());
        assertEquals("columnA", impTable3.getImportTargetColumns().get(0));
        assertEquals("columnB", impTable3.getImportTargetColumns().get(1));
        // 検索条件
        assertNull(impTable3.getSearchCondition());
        // キャッシュ利用有無
        assertEquals(false, impTable3.isUseCache());
        // ロック取得タイプ
        assertEquals(ImportTableLockType.find("3"), impTable3.getLockType());
        // ロック済みの場合の取り扱い
        assertEquals(ImportTableLockedOperation.find("2"), impTable3.getLockedOperation());
        // JavaBeansクラス名
        assertEquals("com.asakusafw.bulkloader.bean.ExporterBean", impTable3.getImportTargetType().getName());
        // HDFS上の出力パス
        assertEquals("/asakusa/import/11/ZZZ", impTable3.getDfsFilePath());

        // Export設定の検証
        // テーブルXXXの検証
        ExportTargetTableBean expTable1 = exportTargetTable.get("XXX");
        // 重複チェックを行うか否か
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

            }
        };
        boolean result = loader.loadCacheBuildParam(targetName, "batch01", "11");
        assertThat(result, is(true));
        assertThat(loader.getImportTargetTables().size(), is(2));
        ImportTargetTableBean target = loader.getImportTargetTables().get("XXX");
        assertThat(target.getCacheId(), is(notNullValue()));
        assertThat(target.getLockType(), is(ImportTableLockType.NONE));
    }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    public void checkImportParamTest02() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.NONE);
        tableBean1.setLockedOperation(ImportTableLockedOperation.FORCE);
        tableBean1.setImportTargetType(this.getClass());
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        ImportTargetTableBean tableBean2 = new ImportTargetTableBean();
        tableBean2.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA2"}));
        tableBean2.setSearchCondition("TEXTDATA2='testdata2-3'");
        tableBean2.setUseCache(false);
        tableBean2.setLockType(ImportTableLockType.find("1"));
        tableBean2.setLockedOperation(ImportTableLockedOperation.find("3"));
        tableBean2.setImportTargetType(this.getClass());
        tableBean2.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET2", tableBean2);


        // テスト実行1
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);

        // 設定変更
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", null}));

        // テスト実行2
        result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);

        // 設定変更
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1"}));
        tableBean2.setImportTargetColumns(Arrays.asList(new String[]{""}));

        // テスト実行3
        result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);
    }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    public void checkImportParamTest03() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.find("0"));
        tableBean1.setLockedOperation(ImportTableLockedOperation.find("1"));
        tableBean1.setImportTargetType(this.getClass());
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        // テスト実行1
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);

        // 設定変更
        tableBean1.setLockType(null);

        // テスト実行2
        result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);
    }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    public void checkImportParamTest04() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.find("1"));
        tableBean1.setLockedOperation(ImportTableLockedOperation.find("0"));
        tableBean1.setImportTargetType(this.getClass());
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        // テスト実行1
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);

        // 設定変更
        tableBean1.setLockedOperation(null);

        // テスト実行2
        result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);
    }
View Full Code Here

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

    public void checkImportParamTest05() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.find("1"));
        tableBean1.setLockedOperation(ImportTableLockedOperation.find("3"));
        tableBean1.setImportTargetType(null);
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        // テスト実行
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.