Package com.asakusafw.runtime.util

Examples of com.asakusafw.runtime.util.VariableTable


    public <V> DataModelSource createSource(
            DataModelDefinition<V> definition,
            TemporaryOutputDescription description,
            TestContext context) throws IOException {
        LOG.info("Retrieving output: {}", description);
        VariableTable variables = createVariables(context);
        checkType(definition, description);
        Configuration conf = configurations.newInstance();
        String resolved = variables.parse(description.getPathPrefix(), false);
        return new TemporaryDataModelSource(conf, definition, resolved);
    }
View Full Code Here


        return new TemporaryDataModelSource(conf, definition, resolved);
    }

    private VariableTable createVariables(TestContext context) {
        assert context != null;
        VariableTable result = new VariableTable();
        result.defineVariables(context.getArguments());
        return result;
    }
View Full Code Here

            FileSystem.closeAll();
        }
    }

    private Path getPath(Configuration conf) {
        VariableTable variables = getPathParser(conf);
        String barePath = getCleanupPath();
        String pathString = variables.parse(barePath, false);
        Path path = new Path(pathString);
        return path;
    }
View Full Code Here

        return path;
    }

    private VariableTable getPathParser(Configuration configuration) {
        assert configuration != null;
        VariableTable variables = new VariableTable(RedefineStrategy.IGNORE);
        variables.defineVariable(VAR_USER, getUser());
        variables.defineVariable(VAR_DEFINITION_ID, getDefinitionId());
        variables.defineVariable(VAR_STAGE_ID, getStageId());
        variables.defineVariable(VAR_BATCH_ID, getBatchId());
        variables.defineVariable(VAR_FLOW_ID, getFlowId());
        variables.defineVariable(VAR_EXECUTION_ID, getExecutionId());
        String arguments = configuration.get(PROP_ASAKUSA_BATCH_ARGS);
        if (arguments == null) {
            LOG.warn(MessageFormat.format(
                    "A mandatory property \"{0}\" does not defined",
                    PROP_ASAKUSA_BATCH_ARGS));
        } else {
            variables.defineVariables(arguments);
        }
        return variables;
    }
View Full Code Here

    public Job createJob(Configuration conf) throws IOException, InterruptedException {
        if (conf == null) {
            throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
        }
        Job job = JobCompatibility.newJob(conf);
        VariableTable variables = getPathParser(job.getConfiguration());
        configureJobInfo(job, variables);
        configureStageInput(job, variables);
        configureStageOutput(job, variables);
        configureShuffle(job, variables);
        configureStageResource(job, variables);
View Full Code Here

                        getClass().getName(),
                        context.getTaskAttemptID()));
            }
            DirectDataSourceRepository repository = HadoopDataSourceUtil.loadRepository(context.getConfiguration());
            String arguments = context.getConfiguration().get(StageConstants.PROP_ASAKUSA_BATCH_ARGS, "");
            VariableTable variables = new VariableTable(VariableTable.RedefineStrategy.IGNORE);
            variables.defineVariables(arguments);

            String path = variables.parse(rawBasePath, false);
            String id = repository.getRelatedId(path);
            OutputAttemptContext outputContext = HadoopDataSourceUtil.createContext(context, id);
            DataFormat<? super T> format = ReflectionUtils.newInstance(dataFormatClass, context.getConfiguration());
            DirectDataSource datasource = repository.getRelatedDataSource(path);
            String basePath = repository.getComponentPath(path);
            String unresolvedResourcePath = rawResourcePath.replaceAll(
                    Pattern.quote("*"),
                    String.format("%04d", context.getTaskAttemptID().getTaskID().getId()));
            String resourcePath = variables.parse(unresolvedResourcePath);
            DataDefinition<? super T> definition = SimpleDataDefinition.newInstance(dataType, format);

            if (log.isDebugEnabled()) {
                log.debug(MessageFormat.format(
                        "Open mapper output (id={0}, basePath={1}, resourcePath={2})",
View Full Code Here

        }
    }

    private VariableTable getPathParser(Configuration configuration) {
        assert configuration != null;
        VariableTable variables = new VariableTable(RedefineStrategy.IGNORE);
        variables.defineVariable(VAR_USER, getUser());
        variables.defineVariable(VAR_DEFINITION_ID, getDefinitionId());
        variables.defineVariable(VAR_STAGE_ID, getStageId());
        variables.defineVariable(VAR_BATCH_ID, getBatchId());
        variables.defineVariable(VAR_FLOW_ID, getFlowId());
        variables.defineVariable(VAR_EXECUTION_ID, getExecutionId());
        String arguments = configuration.get(PROP_ASAKUSA_BATCH_ARGS);
        if (arguments == null) {
            LOG.warn(MessageFormat.format(
                    "Missing configuration \"{0}\" (batch arguments)",
                    PROP_ASAKUSA_BATCH_ARGS));
        } else {
            variables.defineVariables(arguments);
        }

        // replace variables
        configuration.set(PROP_ASAKUSA_BATCH_ARGS, variables.toSerialString());
        return variables;
    }
View Full Code Here

    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        this.recordCounter = JobCompatibility.getTaskOutputRecordCounter(context);
        this.repository = HadoopDataSourceUtil.loadRepository(context.getConfiguration());
        String arguments = context.getConfiguration().get(StageConstants.PROP_ASAKUSA_BATCH_ARGS, "");
        this.variables = new VariableTable(VariableTable.RedefineStrategy.IGNORE);
        variables.defineVariables(arguments);
    }
View Full Code Here

            List<StageInput> inputList) throws IOException {
        assert context != null;
        assert repo != null;
        assert inputList != null;
        String arguments = context.getConfiguration().get(StageConstants.PROP_ASAKUSA_BATCH_ARGS, "");
        VariableTable variables = new VariableTable(VariableTable.RedefineStrategy.IGNORE);
        variables.defineVariables(arguments);

        Map<DirectInputGroup, List<InputPath>> results = new HashMap<DirectInputGroup, List<InputPath>>();
        for (StageInput input : inputList) {
            String fullBasePath = variables.parse(extractBasePath(input));
            String basePath = repo.getComponentPath(fullBasePath);
            FilePattern pattern = extractSearchPattern(context, variables, input);
            Class<?> dataClass = extractDataClass(context, input);
            Class<? extends DataFormat<?>> formatClass = extractFormatClass(context, input);
            DirectInputGroup group = new DirectInputGroup(fullBasePath, dataClass, formatClass);
View Full Code Here

        assert context != null;
        String encoded = context.getConfiguration().getRaw(KEY);
        if (encoded == null) {
            return Collections.emptyList();
        }
        VariableTable table = getVariableTable(context);
        try {
            ByteArrayInputStream source = new ByteArrayInputStream(encoded.getBytes(ASCII));
            DataInputStream input = new DataInputStream(new GZIPInputStream(new Base64InputStream(source)));
            long version = WritableUtils.readVLong(input);
            if (version != SERIAL_VERSION) {
                throw new IOException(MessageFormat.format(
                        "Invalid StageOutput version: framework={0}, saw={1}",
                        SERIAL_VERSION,
                        version));
            }
            List<OutputSpec> results = new ArrayList<OutputSpec>();
            int specCount = WritableUtils.readVInt(input);
            for (int specIndex = 0; specIndex < specCount; specIndex++) {
                String basePath = WritableUtils.readString(input);
                try {
                    basePath = table.parse(basePath);
                } catch (IllegalArgumentException e) {
                    throw new IllegalStateException(MessageFormat.format(
                            "Invalid basePath: {0}",
                            basePath), e);
                }
                int patternCount = WritableUtils.readVInt(input);
                List<String> patterns = new ArrayList<String>();
                for (int patternIndex = 0; patternIndex < patternCount; patternIndex++) {
                    String pattern = WritableUtils.readString(input);
                    try {
                        pattern = table.parse(pattern);
                    } catch (IllegalArgumentException e) {
                        throw new IllegalStateException(MessageFormat.format(
                                "Invalid delete pattern: {0}",
                                pattern), e);
                    }
View Full Code Here

TOP

Related Classes of com.asakusafw.runtime.util.VariableTable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.