Package com.asakusafw.bulkloader.bean

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

192
193
194
195
196
197
198
199
200
        Map<String, ImportTargetTableBean> importTargetTable = loder.getImportTargetTables();

        // 実行結果を検証
        assertTrue(result);
        // テーブルXXXの検証
        ImportTargetTableBean table1 = importTargetTable.get("XXX");
        // 検索条件
        assertEquals("columnA='asakusa' and columnB='hadoop'", table1.getSearchCondition());
    }
View Full Code Here
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
    public void checkImportParamTest01() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.find("1"));
        tableBean1.setLockedOperation(ImportTableLockedOperation.find("3"));
        tableBean1.setImportTargetType(this.getClass());
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        ImportTargetTableBean tableBean2 = new ImportTargetTableBean();
        tableBean2.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA2"}));
        tableBean2.setSearchCondition("TEXTDATA2='testdata2-3'");
        tableBean2.setUseCache(false);
        tableBean2.setLockType(ImportTableLockType.find("2"));
        tableBean2.setLockedOperation(ImportTableLockedOperation.find("1"));
        tableBean2.setImportTargetType(this.getClass());
        tableBean2.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET2", tableBean2);

        ImportTargetTableBean tableBean3 = new ImportTargetTableBean();
        tableBean3.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA3"}));
        tableBean3.setSearchCondition("TEXTDATA2='testdata2-3'");
        tableBean3.setUseCache(false);
        tableBean3.setLockType(ImportTableLockType.find("2"));
        tableBean3.setLockedOperation(ImportTableLockedOperation.find("3"));
        tableBean3.setImportTargetType(this.getClass());
        tableBean3.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET3", tableBean3);

        ImportTargetTableBean tableBean4 = new ImportTargetTableBean();
        tableBean4.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA3"}));
        tableBean4.setSearchCondition("TEXTDATA2='testdata2-3'");
        tableBean4.setUseCache(false);
        tableBean4.setLockType(ImportTableLockType.find("3"));
        tableBean4.setLockedOperation(ImportTableLockedOperation.find("2"));
        tableBean4.setImportTargetType(this.getClass());
        tableBean4.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET4", tableBean4);

        ImportTargetTableBean tableBean5 = new ImportTargetTableBean();
        tableBean5.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA3"}));
        tableBean5.setSearchCondition("TEXTDATA2='testdata2-3'");
        tableBean5.setUseCache(false);
        tableBean5.setLockType(ImportTableLockType.find("3"));
        tableBean5.setLockedOperation(ImportTableLockedOperation.find("3"));
        tableBean5.setImportTargetType(this.getClass());
        tableBean5.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET5", tableBean5);

        // テスト実行
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertTrue(result);
View Full Code Here
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
//        // テストデータの指定
//        String pattern = "patternC01";

        // ImportBeanを生成
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean = new ImportTargetTableBean();
        tableBean.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean.setSearchCondition("1 < 0");
        tableBean.setUseCache(false);
        tableBean.setLockType(ImportTableLockType.TABLE);
        tableBean.setLockedOperation(ImportTableLockedOperation.ERROR);
        tableBean.setImportTargetType(null);
        tableBean.setDfsFilePath(null);
        targetTable.put("IMPORT_TARGET1", tableBean);
        ImportBean bean = createBean(new String[]{jobflowId, executionId, "20101021221015", "3", "5"}, targetTable);

        String jobflowSid = "7";
View Full Code Here
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
        // テストデータの指定
//        String pattern = "patternC01";

        // ImportBeanを生成
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean = new ImportTargetTableBean();
        tableBean.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean.setSearchCondition(null);
        tableBean.setUseCache(false);
        tableBean.setLockType(ImportTableLockType.TABLE);
        tableBean.setLockedOperation(ImportTableLockedOperation.ERROR);
        tableBean.setImportTargetType(null);
        tableBean.setDfsFilePath(null);
        targetTable.put("IMPORT_TARGET1", tableBean);
        ImportBean bean = createBean(new String[]{jobflowId, executionId, "20101021221015", "3", "5"}, targetTable);

        // プロパティを書き換え
        Properties prop = ConfigurationLoader.getProperty();
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
    public void update_cache_rebuild() throws Exception {
        ImportBean bean = createBean();

        Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();

        final ImportTargetTableBean tb1 = new ImportTargetTableBean();
        tb1.setCacheId("tb1");
        tb1.setDfsFilePath("tb1");
        tb1.setImportTargetType(ImportTarget1.class);
        tb1.setImportTargetColumns(Arrays.asList("A"));
        tb1.setSearchCondition("");
        targetTable.put("__TG_TEST1", tb1);

        Connection conn = DBConnection.getConnection();
        final Calendar last = offset(-1);
        try {
            LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn);
            repo.putCacheInfo(new LocalCacheInfo(
                    tb1.getCacheId(),
                    null,
                    last,
                    "__TG_TEST1",
                    tb1.getDfsFilePath()));

        } finally {
            DBConnection.closeConn(conn);
        }

        bean.setTargetTable(targetTable);
        ImportProtocolDecide service = new ImportProtocolDecide() {
            @Override
            protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
                    throws BulkLoaderSystemException {
                return Collections.singletonMap("tb1", new CacheInfo(
                        CacheInfo.FEATURE_VERSION,
                        tb1.getCacheId(),
                        last,
                        "__TG_TEST1",
                        tb1.getImportTargetColumns(),
                        tb1.getImportTargetType().getName(),
                        new ImportTarget1().__tgc__DataModelVersion()));
            }
        };
        service.execute(bean);

        assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.UPDATE_CACHE));
        assertThat(tb1.getImportProtocol().getLocation(), is(tb1.getDfsFilePath()));
        assertThat(tb1.getStartTimestamp(), is(notNullValue()));
        CacheInfo info = tb1.getImportProtocol().getInfo();
        assertThat(info, is(notNullValue()));

        assertThat(info.getId(), is("tb1"));
        assertThat(info.getFeatureVersion(), is(CacheInfo.FEATURE_VERSION));
        assertThat(info.getTimestamp(), is(not(nullValue())));
        assertThat(info.getTableName(), is("__TG_TEST1"));
        assertThat(info.getColumnNames(), is((Object) new HashSet<String>(tb1.getImportTargetColumns())));
        assertThat(info.getModelClassName(), is(ImportTarget1.class.getName()));
        assertThat(info.getModelClassVersion(), is(new ImportTarget1().__tgc__DataModelVersion()));
    }
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
    public void update_cache_db_rollback() throws Exception {
        ImportBean bean = createBean();

        Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();

        final ImportTargetTableBean tb1 = new ImportTargetTableBean();
        tb1.setCacheId("tb1");
        tb1.setDfsFilePath("tb1");
        tb1.setImportTargetType(ImportTarget1.class);
        tb1.setImportTargetColumns(Arrays.asList("A"));
        tb1.setSearchCondition("");
        targetTable.put("__TG_TEST1", tb1);

        Connection conn = DBConnection.getConnection();
        try {
            LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn);
            repo.putCacheInfo(new LocalCacheInfo(
                    tb1.getCacheId(),
                    null,
                    null,
                    "__TG_TEST1",
                    tb1.getDfsFilePath()));

        } finally {
            DBConnection.closeConn(conn);
        }

        bean.setTargetTable(targetTable);
        ImportProtocolDecide service = new ImportProtocolDecide() {
            @Override
            protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
                    throws BulkLoaderSystemException {
                return Collections.singletonMap("tb1", new CacheInfo(
                        CacheInfo.FEATURE_VERSION,
                        tb1.getCacheId(),
                        offset(+1),
                        "__TG_TEST1",
                        tb1.getImportTargetColumns(),
                        tb1.getImportTargetType().getName(),
                        new ImportTarget1().__tgc__DataModelVersion()));
            }
        };
        service.execute(bean);

        assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE));
    }
View Full Code Here
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
    public void update_cache_dfs_rollback() throws Exception {
        ImportBean bean = createBean();

        Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();

        final ImportTargetTableBean tb1 = new ImportTargetTableBean();
        tb1.setCacheId("tb1");
        tb1.setDfsFilePath("tb1");
        tb1.setImportTargetType(ImportTarget1.class);
        tb1.setImportTargetColumns(Arrays.asList("A"));
        tb1.setSearchCondition("");
        targetTable.put("__TG_TEST1", tb1);

        Connection conn = DBConnection.getConnection();
        try {
            LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn);
            repo.putCacheInfo(new LocalCacheInfo(
                    tb1.getCacheId(),
                    null,
                    offset(-1),
                    "__TG_TEST1",
                    tb1.getDfsFilePath()));

        } finally {
            DBConnection.closeConn(conn);
        }

        bean.setTargetTable(targetTable);
        ImportProtocolDecide service = new ImportProtocolDecide() {
            @Override
            protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
                    throws BulkLoaderSystemException {
                return Collections.singletonMap("tb1", new CacheInfo(
                        CacheInfo.FEATURE_VERSION,
                        tb1.getCacheId(),
                        offset(-2),
                        "__TG_TEST1",
                        tb1.getImportTargetColumns(),
                        tb1.getImportTargetType().getName(),
                        new ImportTarget1().__tgc__DataModelVersion()));
            }
        };
        service.execute(bean);

        assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE));
    }
View Full Code Here
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
    public void update_cache_broken_local() throws Exception {
        ImportBean bean = createBean();

        Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();

        final ImportTargetTableBean tb1 = new ImportTargetTableBean();
        tb1.setCacheId("tb1");
        tb1.setDfsFilePath("tb1");
        tb1.setImportTargetType(ImportTarget1.class);
        tb1.setImportTargetColumns(Arrays.asList("A"));
        tb1.setSearchCondition("");
        targetTable.put("__TG_TEST1", tb1);

        bean.setTargetTable(targetTable);
        ImportProtocolDecide service = new ImportProtocolDecide() {
            @Override
            protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
                    throws BulkLoaderSystemException {
                return Collections.singletonMap("tb1", new CacheInfo(
                        CacheInfo.FEATURE_VERSION,
                        tb1.getCacheId(),
                        offset(-1),
                        "__TG_TEST1",
                        tb1.getImportTargetColumns(),
                        tb1.getImportTargetType().getName(),
                        new ImportTarget1().__tgc__DataModelVersion()));
            }
        };
        service.execute(bean);

        assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE));
    }
View Full Code Here
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
    public void update_cache_broken_remote() throws Exception {
        ImportBean bean = createBean();

        Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();

        final ImportTargetTableBean tb1 = new ImportTargetTableBean();
        tb1.setCacheId("tb1");
        tb1.setDfsFilePath("tb1");
        tb1.setImportTargetType(ImportTarget1.class);
        tb1.setImportTargetColumns(Arrays.asList("A"));
        tb1.setSearchCondition("");
        targetTable.put("__TG_TEST1", tb1);

        Connection conn = DBConnection.getConnection();
        try {
            LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn);
            repo.putCacheInfo(new LocalCacheInfo(
                    tb1.getCacheId(),
                    null,
                    null,
                    "__TG_TEST1",
                    tb1.getDfsFilePath()));

        } finally {
            DBConnection.closeConn(conn);
        }

        bean.setTargetTable(targetTable);
        ImportProtocolDecide service = new ImportProtocolDecide() {
            @Override
            protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
                    throws BulkLoaderSystemException {
                return Collections.emptyMap();
            }
        };
        service.execute(bean);

        assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE));
    }
View Full Code Here
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
    public void update_cache_feature_changed() throws Exception {
        ImportBean bean = createBean();

        Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();

        final ImportTargetTableBean tb1 = new ImportTargetTableBean();
        tb1.setCacheId("tb1");
        tb1.setDfsFilePath("tb1");
        tb1.setImportTargetType(ImportTarget1.class);
        tb1.setImportTargetColumns(Arrays.asList("A"));
        tb1.setSearchCondition("");
        targetTable.put("__TG_TEST1", tb1);

        Connection conn = DBConnection.getConnection();
        try {
            LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn);
            repo.putCacheInfo(new LocalCacheInfo(
                    tb1.getCacheId(),
                    null,
                    null,
                    "__TG_TEST1",
                    tb1.getDfsFilePath()));

        } finally {
            DBConnection.closeConn(conn);
        }

        bean.setTargetTable(targetTable);
        ImportProtocolDecide service = new ImportProtocolDecide() {
            @Override
            protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
                    throws BulkLoaderSystemException {
                return Collections.singletonMap("tb1", new CacheInfo(
                        CacheInfo.FEATURE_VERSION + "+",
                        tb1.getCacheId(),
                        offset(-1),
                        "__TG_TEST1",
                        tb1.getImportTargetColumns(),
                        tb1.getImportTargetType().getName(),
                        new ImportTarget1().__tgc__DataModelVersion()));
            }
        };
        service.execute(bean);

        assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE));
    }
View Full Code Here
TOP

Related Classes of com.asakusafw.bulkloader.bean.ImportTargetTableBean

  • com.asakusafw.bulkloader.cache.BuildCache
  • com.asakusafw.bulkloader.cache.GetCacheInfoLocal
  • com.asakusafw.bulkloader.cache.GetCacheInfoLocalTest
  • com.asakusafw.bulkloader.common.JobFlowParamLoader
  • com.asakusafw.bulkloader.common.JobFlowParamLoaderTest
  • com.asakusafw.bulkloader.exporter.LockReleaseTest
  • com.asakusafw.bulkloader.extractor.DfsFileImport
  • com.asakusafw.bulkloader.extractor.DfsFileImportTest
  • com.asakusafw.bulkloader.importer.ImportFileCreate
  • com.asakusafw.bulkloader.importer.ImportFileCreateTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.