Package com.asakusafw.runtime.directio

Examples of com.asakusafw.runtime.directio.DirectDataSource

471
472
473
474
475
476
477
478
479
480
481
482
                            taskContext.getJobID(),
                            taskContext.getTaskAttemptID()));
                }
                OutputAttemptContext context = HadoopDataSourceUtil.createContext(taskContext, id);
                try {
                    DirectDataSource repo = repository.getRelatedDataSource(containerPath);
                    repo.cleanupAttemptOutput(context);
                } catch (IOException e) {
                    LOG.error(MessageFormat.format(
                            "Failed directio task cleanup: datasource={0} (job={1}, task={2})",
                            id,
                            taskContext.getJobID(),
View Full Code Here
514
515
516
517
518
519
520
521
522
523
524
525
                            id,
                            jobContext.getJobID()));
                }
                OutputTransactionContext context = HadoopDataSourceUtil.createContext(jobContext, id);
                try {
                    DirectDataSource repo = repository.getRelatedDataSource(containerPath);
                    repo.setupTransactionOutput(context);
                } catch (IOException e) {
                    LOG.error(MessageFormat.format(
                            "Failed directio job setup: datasource={0} (job={1})",
                            id,
                            jobContext.getJobID()), e);
View Full Code Here
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
                    continue;
                }
                String id = repository.getRelatedId(spec.basePath);
                OutputTransactionContext context = HadoopDataSourceUtil.createContext(jobContext, id);
                try {
                    DirectDataSource repo = repository.getRelatedDataSource(spec.basePath);
                    String basePath = repository.getComponentPath(spec.basePath);
                    for (String pattern : spec.deletePatterns) {
                        FilePattern resources = FilePattern.compile(pattern);
                        if (LOG.isInfoEnabled()) {
                            LOG.info(MessageFormat.format(
                                    "Deleting output: datasource={0}, basePath={1}, pattern={2}",
                                    id,
                                    basePath,
                                    pattern));
                        }
                        boolean succeed = repo.delete(basePath, resources, true, context.getCounter());
                        if (LOG.isDebugEnabled()) {
                            LOG.debug(MessageFormat.format(
                                    "Deleted output (succeed={3}): datasource={0}, basePath={1}, pattern={2}",
                                    id,
                                    basePath,
View Full Code Here
754
755
756
757
758
759
760
761
762
763
764
765
                            id,
                            jobContext.getJobID()));
                }
                OutputTransactionContext context = HadoopDataSourceUtil.createContext(jobContext, id);
                try {
                    DirectDataSource repo = repository.getRelatedDataSource(containerPath);
                    repo.commitTransactionOutput(context);
                } catch (IOException e) {
                    LOG.error(MessageFormat.format(
                            "Failed directio job rollforward: datasource={0} (job={1})",
                            id,
                            jobContext.getJobID()), e);
View Full Code Here
784
785
786
787
788
789
790
791
792
793
794
795
                            id,
                            jobContext.getJobID()));
                }
                OutputTransactionContext context = HadoopDataSourceUtil.createContext(jobContext, id);
                try {
                    DirectDataSource repo = repository.getRelatedDataSource(containerPath);
                    repo.cleanupTransactionOutput(context);
                } catch (IOException e) {
                    LOG.error(MessageFormat.format(
                            "Failed directio job cleanup: datasource={0} (job={1})",
                            id,
                            jobContext.getJobID()), e);
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
        Configuration conf = new Configuration();
        conf.set(key("testing"), MockHadoopDataSource.class.getName());
        conf.set(key("testing", "path"), "testing");
        conf.set(key("testing", "hello"), "world");
        DirectDataSourceRepository repo = HadoopDataSourceUtil.loadRepository(conf);
        DirectDataSource ds = repo.getRelatedDataSource("testing");
        assertThat(ds, instanceOf(MockHadoopDataSource.class));
        MockHadoopDataSource mock = (MockHadoopDataSource) ds;
        assertThat(mock.conf, is(notNullValue()));
        assertThat(mock.profile.getPath(), is("testing"));
    }
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
231
232
233
        }
        DirectDataSourceRepository repo = getRepository();
        for (String containerPath : repo.getContainerPaths()) {
            String datasourceId = repo.getRelatedId(containerPath);
            try {
                DirectDataSource datasource = repo.getRelatedDataSource(containerPath);
                OutputTransactionContext context = HadoopDataSourceUtil.createContext(executionId, datasourceId);
                datasource.commitTransactionOutput(context);
                datasource.cleanupTransactionOutput(context);
            } catch (IOException e) {
                succeed = false;
                LOG.error(MessageFormat.format(
                        "Failed to apply transaction (datastoreId={0}, executionId={1})",
                        datasourceId,
View Full Code Here
290
291
292
293
294
295
296
297
298
299
300
301
302
        }
        DirectDataSourceRepository repo = getRepository();
        for (String containerPath : repo.getContainerPaths()) {
            String datasourceId = repo.getRelatedId(containerPath);
            try {
                DirectDataSource datasource = repo.getRelatedDataSource(containerPath);
                OutputTransactionContext context = HadoopDataSourceUtil.createContext(executionId, datasourceId);
                datasource.cleanupTransactionOutput(context);
            } catch (IOException e) {
                succeed = false;
                LOG.error(MessageFormat.format(
                        "Failed to abort transaction (datastoreId={0}, executionId={1})",
                        datasourceId,
View Full Code Here
TOP

Related Classes of com.asakusafw.runtime.directio.DirectDataSource

  • com.asakusafw.directio.tools.DirectIoCommandTestRoot
  • com.asakusafw.directio.tools.DirectIoDelete
  • com.asakusafw.directio.tools.DirectIoList
  • com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor
  • com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditorTest
  • com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtilTest
  • com.asakusafw.runtime.stage.directio.AbstractNoReduceDirectOutputMapper
  • com.asakusafw.runtime.stage.directio.DirectOutputReducer
  • com.asakusafw.runtime.stage.input.BridgeInputFormat
  • com.asakusafw.runtime.stage.output.BridgeOutputFormat$BridgeOutputCommitter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.