Examples of HCatSchema

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
      partitionVals = new HashMap<String, String>(1);
      partitionVals.put(s[0], val);
    }
    HCatOutputFormat.setOutput(job, OutputJobInfo.create(dbName,
      outputTableName, partitionVals));
    HCatSchema s = HCatInputFormat.getTableSchema(job);
    // Build the schema for this table, which is slightly different than the
    // schema for the input table
    List<HCatFieldSchema> fss = new ArrayList<HCatFieldSchema>(3);
    fss.add(s.get(0));
    fss.add(s.get(1));
    fss.add(s.get(3));
    HCatOutputFormat.setSchema(job, new HCatSchema(fss));
    job.setOutputFormatClass(HCatOutputFormat.class);
    return (job.waitForCompletion(true) ? 0 : 1);
  }
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

160
161
162
163
164
165
166
167
168
169
170
171
172
      return null;
    }

    Properties props = UDFContext.getUDFContext().getUDFProperties(
      classForUDFCLookup, new String[]{signature});
    HCatSchema hcatTableSchema = (HCatSchema) props.get(HCatConstants.HCAT_TABLE_SCHEMA);

    ArrayList<HCatFieldSchema> fcols = new ArrayList<HCatFieldSchema>();
    for (RequiredField rf : fields) {
      fcols.add(hcatTableSchema.getFields().get(rf.getIndex()));
    }
    return new HCatSchema(fcols);
  }
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

101
102
103
104
105
106
107
108
109
110
111
    job.setOutputKeyClass(WritableComparable.class);
    job.setOutputValueClass(DefaultHCatRecord.class);
    job.setNumReduceTasks(0);
    HCatOutputFormat.setOutput(job, OutputJobInfo.create(dbName,
      outputTableName, null));
    HCatSchema s = HCatInputFormat.getTableSchema(job);
    System.err.println("INFO: output schema explicitly set for writing:"
      + s);
    HCatOutputFormat.setSchema(job, s);
    job.setOutputFormatClass(HCatOutputFormat.class);
    return (job.waitForCompletion(true) ? 0 : 1);
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

127
128
129
130
131
132
133
134
135
136
137
        // the table
        // information passed to HCatOutputFormat was not right
        throw new PigException(he.getMessage(),
          PigHCatUtil.PIG_EXCEPTION_CODE, he);
      }
      HCatSchema hcatTblSchema = HCatOutputFormat.getTableSchema(job);
      try {
        doSchemaValidations(pigSchema, hcatTblSchema);
      } catch (HCatException he) {
        throw new FrontendException(he.getMessage(), PigHCatUtil.PIG_EXCEPTION_CODE, he);
      }
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

250
251
252
253
254
255
256
257
258
259
260
261
262
263
      context.write(NullWritable.get(), record);
    }
  }

  private HCatSchema getSchema() throws HCatException {
    HCatSchema schema = new HCatSchema(new ArrayList<HCatFieldSchema>());
    schema.append(new HCatFieldSchema("a0", HCatFieldSchema.Type.INT,
        ""));
    schema.append(new HCatFieldSchema("a1",
        HCatFieldSchema.Type.STRING, ""));
    schema.append(new HCatFieldSchema("a2",
        HCatFieldSchema.Type.STRING, ""));
    return schema;
  }
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
    for (PartInfo partitionInfo : partitionInfoList) {
      jobConf = HCatUtil.getJobConfFromContext(jobContext);
      setInputPath(jobConf, partitionInfo.getLocation());
      Map<String, String> jobProperties = partitionInfo.getJobProperties();

      HCatSchema allCols = new HCatSchema(new LinkedList<HCatFieldSchema>());
      for (HCatFieldSchema field :
        inputJobInfo.getTableInfo().getDataColumns().getFields())
        allCols.append(field);
      for (HCatFieldSchema field :
        inputJobInfo.getTableInfo().getPartitionColumns().getFields())
        allCols.append(field);

      HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf);

      storageHandler = HCatUtil.getStorageHandler(
        jobConf, partitionInfo);
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

204
205
206
207
208
209
210
211
212
213
214
215
216
217
  /**
   * gets values for fields requested by output schema which will not be in the data
   */
  private static Map<String, String> getColValsNotInDataColumns(HCatSchema outputSchema,
                                  PartInfo partInfo) {
    HCatSchema dataSchema = partInfo.getPartitionSchema();
    Map<String, String> vals = new HashMap<String, String>();
    for (String fieldName : outputSchema.getFieldNames()) {
      if (dataSchema.getPosition(fieldName) == null) {
        // this entry of output is not present in the output schema
        // so, we first check the table schema to see if it is a part col

        if (partInfo.getPartitionValues().containsKey(fieldName)) {
          vals.put(fieldName, partInfo.getPartitionValues().get(fieldName));
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

243
244
245
246
247
248
249
250
251
252
253
254
255
256
   *                     for the current context
   */
  public static HCatSchema getTableSchema(Configuration conf)
    throws IOException {
    InputJobInfo inputJobInfo = getJobInfo(conf);
    HCatSchema allCols = new HCatSchema(new LinkedList<HCatFieldSchema>());
    for (HCatFieldSchema field :
      inputJobInfo.getTableInfo().getDataColumns().getFields())
      allCols.append(field);
    for (HCatFieldSchema field :
      inputJobInfo.getTableInfo().getPartitionColumns().getFields())
      allCols.append(field);
    return allCols;
  }
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

127
128
129
130
131
132
133
134
135
136
137
138
139
140
      context.write(NullWritable.get(), record);
    }
  }

  private HCatSchema getSchema() throws HCatException {
    HCatSchema schema = new HCatSchema(new ArrayList<HCatFieldSchema>());
    schema.append(new HCatFieldSchema("a0", HCatFieldSchema.Type.INT,
        ""));
    schema.append(new HCatFieldSchema("a1",
        HCatFieldSchema.Type.STRING, ""));
    schema.append(new HCatFieldSchema("a2",
        HCatFieldSchema.Type.STRING, ""));
    return schema;
  }
View Full Code Here

Examples of org.apache.hive.hcatalog.data.schema.HCatSchema

118
119
120
121
122
123
124
125
126
127
128
      dbName, tableName, null));
    // initialize HCatOutputFormat
    HCatOutputFormat.setOutput(job, OutputJobInfo.create(
      dbName, outputTableName, outputPartitionKvps));
    // test with and without specifying schema randomly
    HCatSchema s = HCatInputFormat.getTableSchema(job);
    System.err.println("INFO: output schema explicitly set for writing:" + s);
    HCatOutputFormat.setSchema(job, s);

    job.setInputFormatClass(HCatInputFormat.class);
    job.setOutputFormatClass(HCatOutputFormat.class);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.