Examples of HCatTableInfo

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

99
100
101
102
103
104
105
106
107
108
109
    // hbase-default.xml and hbase-site.xml
    Map<String, String> tableJobProperties = tableDesc.getJobProperties();
    String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO);
    try {
      InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);
      HCatTableInfo tableInfo = inputJobInfo.getTableInfo();
      String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
      jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);

      Configuration jobConf = getJobConf();
      addResources(jobConf, jobProperties);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    // In case of bulk mode, populate intermediate output location
    Map<String, String> tableJobProperties = tableDesc.getJobProperties();
    String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_OUTPUT_INFO);
    try {
      OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString);
      HCatTableInfo tableInfo = outputJobInfo.getTableInfo();
      String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
      jobProperties.put(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, qualifiedTableName);
      jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName);

      Configuration jobConf = getJobConf();
      addResources(jobConf, jobProperties);

      Configuration copyOfConf = new Configuration(jobConf);
      HBaseConfiguration.addHbaseResources(copyOfConf);

      String txnString = outputJobInfo.getProperties().getProperty(
        HBaseConstants.PROPERTY_WRITE_TXN_KEY);
      Transaction txn = null;
      if (txnString == null) {
        txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo,
          RevisionManagerConfiguration.create(copyOfConf));
        String serializedTxn = HCatUtil.serialize(txn);
        outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
          serializedTxn);
      } else {
        txn = (Transaction) HCatUtil.deserialize(txnString);
      }
      if (isBulkMode(outputJobInfo)) {
        String tableLocation = tableInfo.getTableLocation();
        String location = new Path(tableLocation, "REVISION_" + txn.getRevisionNumber())
          .toString();
        outputJobInfo.getProperties().setProperty(PROPERTY_INT_OUTPUT_LOCATION, location);
        // We are writing out an intermediate sequenceFile hence
        // location is not passed in OutputJobInfo.getLocation()
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

261
262
263
264
265
266
267
268
269
270
271
    TextInputFormat.setInputPaths(job, path);

    job.setOutputFormatClass(HCatOutputFormat.class);

    HCatTableInfo outputInfo = HCatTableInfo.getOutputTableInfo(thriftUri, null, dbName, tableName, partitionValues);
    HCatOutputFormat.setOutput(job, outputInfo);

    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(DefaultHCatRecord.class);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

298
299
300
301
302
303
304
305
306
307
308
    // input/output settings
    job.setInputFormatClass(HCatInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    HCatTableInfo inputInfo = HCatTableInfo.getInputTableInfo(
          thriftUri, null, dbName, tableName, filter);
    HCatInputFormat.setInput(job, inputInfo);

    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(Text.class);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

331
332
333
334
335
336
337
338
339
340
    // input/output settings
    job.setInputFormatClass(HCatInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    HCatTableInfo inputInfo = HCatTableInfo.getInputTableInfo(thriftUri, null, dbName, tableName);
    HCatInputFormat.setInput(job, inputInfo);

    return HCatInputFormat.getTableSchema(job);
  }
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

140
141
142
143
144
145
146
147
148
149
150
    Job job = new Job(conf, "test outputformat");

    Map<String, String> partitionValues = new HashMap<String, String>();
    partitionValues.put("colname", "p1");
    //null server url means local mode
    HCatTableInfo info = HCatTableInfo.getOutputTableInfo(null, null, dbName, tblName, partitionValues);

    HCatOutputFormat.setOutput(job, info);
    OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(job);

    assertNotNull(jobInfo.getTableInfo());
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

70
71
72
73
74
75
76
77
78
79
80
  public void setStoreLocation(String location, Job job) throws IOException {

    Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[]{sign});

    String[] userStr = location.split("\\.");
    HCatTableInfo tblInfo;
    if(userStr.length == 2) {
      tblInfo = HCatTableInfo.getOutputTableInfo(PigHCatUtil.getHCatServerUri(job),
          PigHCatUtil.getHCatServerPrincipal(job), userStr[0],userStr[1],partitions);
    } else {
      tblInfo = HCatTableInfo.getOutputTableInfo(PigHCatUtil.getHCatServerUri(job),
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

93
94
95
96
97
98
99
100
101
102
103
        // hbase-default.xml and hbase-site.xml
        Map<String, String> tableJobProperties = tableDesc.getJobProperties();
        String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO);
        try {
            InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);
            HCatTableInfo tableInfo = inputJobInfo.getTableInfo();
            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
            jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);

            Configuration jobConf = getJobConf();
            addResources(jobConf, jobProperties);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
        // In case of bulk mode, populate intermediate output location
        Map<String, String> tableJobProperties = tableDesc.getJobProperties();
        String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_OUTPUT_INFO);
        try {
            OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString);
            HCatTableInfo tableInfo = outputJobInfo.getTableInfo();
            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
            jobProperties.put(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, qualifiedTableName);
            jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName);

            Configuration jobConf = getJobConf();
            addResources(jobConf, jobProperties);

            Configuration copyOfConf = new Configuration(jobConf);
            HBaseConfiguration.addHbaseResources(copyOfConf);

            String txnString = outputJobInfo.getProperties().getProperty(
                HBaseConstants.PROPERTY_WRITE_TXN_KEY);
            Transaction txn = null;
            if (txnString == null) {
                txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo,
                    RevisionManagerConfiguration.create(copyOfConf));
                String serializedTxn = HCatUtil.serialize(txn);
                outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
                    serializedTxn);
            } else {
                txn = (Transaction) HCatUtil.deserialize(txnString);
            }
            if (isBulkMode(outputJobInfo)) {
                String tableLocation = tableInfo.getTableLocation();
                String location = new Path(tableLocation, "REVISION_" + txn.getRevisionNumber())
                    .toString();
                outputJobInfo.getProperties().setProperty(PROPERTY_INT_OUTPUT_LOCATION, location);
                // We are writing out an intermediate sequenceFile hence
                // location is not passed in OutputJobInfo.getLocation()
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.HCatTableInfo

91
92
93
94
95
96
97
98
99
100
101
        // hbase-default.xml and hbase-site.xml
        Map<String, String> tableJobProperties = tableDesc.getJobProperties();
        String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO);
        try {
            InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);
            HCatTableInfo tableInfo = inputJobInfo.getTableInfo();
            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
            jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);

            Configuration jobConf = getJobConf();
            addHbaseResources(jobConf, jobProperties);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.