Examples of OutputJobInfo

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

206
207
208
209
210
211
212
213
214
        }
        return txn;
    }

    static Transaction getWriteTransaction(Configuration conf) throws IOException {
        OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
        return (Transaction) HCatUtil.deserialize(outputJobInfo.getProperties()
            .getProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY));
    }
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

212
213
214
215
216
217
218
219
220
        return (Transaction) HCatUtil.deserialize(outputJobInfo.getProperties()
            .getProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY));
    }

    static void setWriteTransaction(Configuration conf, Transaction txn) throws IOException {
        OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
        outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, HCatUtil.serialize(txn));
        conf.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo));
    }
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

56
57
58
59
60
61
62
63
64
65
66
        super(config, sp);
    }

    @Override
    public WriterContext prepareWrite() throws HCatException {
        OutputJobInfo jobInfo = OutputJobInfo.create(we.getDbName(),
            we.getTableName(), we.getPartitionKVs());
        Job job;
        try {
            job = new Job(conf);
            HCatOutputFormat.setOutput(job, jobInfo);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
        // Populate RM transaction in OutputJobInfo
        // In case of bulk mode, populate intermediate output location
        Map<String, String> tableJobProperties = tableDesc.getJobProperties();
        String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_OUTPUT_INFO);
        try {
            OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString);
            HCatTableInfo tableInfo = outputJobInfo.getTableInfo();
            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
            jobProperties.put(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, qualifiedTableName);
            jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName);

            Configuration jobConf = getJobConf();
            addResources(jobConf, jobProperties);

            Configuration copyOfConf = new Configuration(jobConf);
            HBaseConfiguration.addHbaseResources(copyOfConf);

            String txnString = outputJobInfo.getProperties().getProperty(
                HBaseConstants.PROPERTY_WRITE_TXN_KEY);
            Transaction txn = null;
            if (txnString == null) {
                txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo,
                    RevisionManagerConfiguration.create(copyOfConf));
                String serializedTxn = HCatUtil.serialize(txn);
                outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
                    serializedTxn);
            } else {
                txn = (Transaction) HCatUtil.deserialize(txnString);
            }
            if (isBulkMode(outputJobInfo)) {
                String tableLocation = tableInfo.getTableLocation();
                String location = new Path(tableLocation, "REVISION_" + txn.getRevisionNumber())
                    .toString();
                outputJobInfo.getProperties().setProperty(PROPERTY_INT_OUTPUT_LOCATION, location);
                // We are writing out an intermediate sequenceFile hence
                // location is not passed in OutputJobInfo.getLocation()
                // TODO replace this with a mapreduce constant when available
                jobProperties.put("mapred.output.dir", location);
                jobProperties.put("mapred.output.committer.class", HBaseBulkOutputCommitter.class.getName());
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
            if (crd != null) {
                job.getCredentials().addAll(crd);
            }
        } else {
            Job clone = new Job(job.getConfiguration());
            OutputJobInfo outputJobInfo;
            if (userStr.length == 2) {
                outputJobInfo = OutputJobInfo.create(userStr[0], userStr[1], partitions);
            } else if (userStr.length == 1) {
                outputJobInfo = OutputJobInfo.create(null, userStr[0], partitions);
            } else {
                throw new FrontendException("location " + location
                    + " is invalid. It must be of the form [db.]table",
                    PigHCatUtil.PIG_EXCEPTION_CODE);
            }
            Schema schema = (Schema) ObjectSerializer.deserialize(udfProps.getProperty(PIG_SCHEMA));
            if (schema != null) {
                pigSchema = schema;
            }
            if (pigSchema == null) {
                throw new FrontendException(
                    "Schema for data cannot be determined.",
                    PigHCatUtil.PIG_EXCEPTION_CODE);
            }
            String externalLocation = (String) udfProps.getProperty(HCatConstants.HCAT_PIG_STORER_EXTERNAL_LOCATION);
            if (externalLocation != null) {
                outputJobInfo.setLocation(externalLocation);
            }
            try {
                HCatOutputFormat.setOutput(job, outputJobInfo);
            } catch (HCatException he) {
                // pass the message to the user - essentially something about
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

211
212
213
214
215
216
217
218
219
220
221
222
223
        job.setOutputCommitter(HBaseBulkOutputCommitter.class);

        //manually create transaction
        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
        try {
            OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null);
            Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName));
            outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
                HCatUtil.serialize(txn));
            job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO,
                HCatUtil.serialize(outputJobInfo));
        } finally {
            rm.close();
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

384
385
386
387
388
389
390
391
392
393
394
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.setInputPaths(job, inputPath);


        job.setOutputFormatClass(HCatOutputFormat.class);
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null);
        HCatOutputFormat.setOutput(job, outputJobInfo);

        job.setMapOutputKeyClass(BytesWritable.class);
        job.setMapOutputValueClass(HCatRecord.class);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

476
477
478
479
480
481
482
483
484
485
486
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.setInputPaths(job, inputPath);


        job.setOutputFormatClass(HCatOutputFormat.class);
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null);
        HCatOutputFormat.setOutput(job, outputJobInfo);

        job.setMapOutputKeyClass(BytesWritable.class);
        job.setMapOutputValueClass(HCatRecord.class);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

548
549
550
551
552
553
554
555
556
557
558
            os.write(Bytes.toBytes(data[i] + "\n"));
            os.close();
        }

        Path workingDir = new Path(methodTestDir, "mr_abort");
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
            tableName, null);
        Job job = configureJob(testName,
            conf, workingDir, MapWriteAbortTransaction.class,
            outputJobInfo, inputPath);
        assertFalse(job.waitForCompletion(true));
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

153
154
155
156
157
158
159
160
161
162
163
164
165
    }

    public static class MapHCatWrite extends Mapper<LongWritable, Text, BytesWritable, HCatRecord> {
        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
            HCatRecord record = new DefaultHCatRecord(3);
            HCatSchema schema = jobInfo.getOutputSchema();
            String vals[] = value.toString().split(",");
            record.setInteger("key", schema, Integer.parseInt(vals[0]));
            for (int i = 1; i < vals.length; i++) {
                String pair[] = vals[i].split(":");
                record.set(pair[0], schema, pair[1]);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.