Examples of HiveOutputDescription

Examples of com.facebook.giraph.hive.output.HiveOutputDescription

105
106
107
108
109
110
111
  /** Create a new runner */
  public HiveGiraphRunner() {
    conf = new HiveConf(getClass());
    hiveVertexInputDescription = new HiveInputDescription();
    hiveEdgeInputDescription = new HiveInputDescription();
    hiveOutputDescription = new HiveOutputDescription();
  }
View Full Code Here

Examples of com.facebook.giraph.hive.output.HiveOutputDescription

70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
  public static final List<MapWritable> mapperData2 = ImmutableList.of(row3, row4);

  private HiveTools() {}

  public static void setupJob(Configuration conf) throws IOException {
    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.setDbName("default");
    outputDesc.setTableName("hive_io_test");
    Map<String, String> partitionValues = ImmutableMap.of("ds", "2013-04-01");
    outputDesc.setPartitionValues(partitionValues);
    LOG.info("Writing to " + outputDesc);
    try {
      HiveApiOutputFormat.initProfile(conf, outputDesc, SAMPLE_PROFILE_ID);
    } catch (TException e) {
      LOG.fatal("Failed to initialize profile " + outputDesc);
View Full Code Here

Examples of com.facebook.giraph.hive.output.HiveOutputDescription

41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
  TBLPROPERTIES ('RETENTION_PLATINUM'='90')
*/
public class WritingTool extends Configured implements Tool {
  @Override
  public int run(String[] args) throws Exception {
    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.setDbName("default");
    outputDesc.setTableName("hive_io_test");
    Map<String, String> partitionValues = ImmutableMap.of("ds", "2013-04-01");
    outputDesc.setPartitionValues(partitionValues);
    HiveApiOutputFormat.initProfile(getConf(), outputDesc, SAMPLE_PROFILE_ID);

    Job job = new Job(getConf(), "hive-io-writing");
    job.setMapperClass(SampleMapper.class);
    job.setInputFormatClass(SampleInputFormat.class);
View Full Code Here

Examples of com.facebook.giraph.hive.output.HiveOutputDescription

96
97
98
99
100
101
102
103
104
105
   * Read user's output description from Configuration
   *
   * @return HiveOutputDescription
   */
  public HiveOutputDescription readOutputDescription() {
    HiveOutputDescription hod = new HiveOutputDescription();
    String value = conf.get(getOutputDescriptionKey());
    Writables.readFieldsFromEncodedStr(value, hod);
    return hod;
  }
View Full Code Here

Examples of com.facebook.giraph.hive.output.HiveOutputDescription

70
71
72
73
74
75
76
77
78
79
80
  public void commitJob(JobContext jobContext) throws IOException {
    baseCommitter.commitJob(jobContext);

    Configuration conf = jobContext.getConfiguration();
    OutputConf outputConf = new OutputConf(conf, profileId);
    HiveOutputDescription outputDesc = outputConf.readOutputDescription();
    OutputInfo outputInfo = outputConf.readOutputTableInfo();
    if (outputInfo.hasPartitionInfo()) {
      registerPartitions(conf, outputDesc, outputInfo);
    } else {
      noPartitionsCopyData(conf, outputInfo);
View Full Code Here

Examples of com.facebook.hiveio.output.HiveOutputDescription

68
69
70
71
72
73
74
75
76
77
78
79
80
   * Create HiveOutputDescription from Configuration
   *
   * @return HiveOutputDescription
   */
  private HiveOutputDescription makeOutputDesc() {
    HiveOutputDescription outputDesc = new HiveOutputDescription();
    HiveTableDesc tableDesc = outputDesc.getTableDesc();
    tableDesc.setDatabaseName(HIVE_VERTEX_OUTPUT_DATABASE.get(getConf()));
    tableDesc.setTableName(HIVE_VERTEX_OUTPUT_TABLE.get(getConf()));
    outputDesc.setPartitionValues(
        parsePartitionValues(HIVE_VERTEX_OUTPUT_PARTITION.get(getConf())));
    return outputDesc;
  }
View Full Code Here

Examples of com.facebook.hiveio.output.HiveOutputDescription

113
114
115
116
117
118
119
120
121
122
123
124
125
126
   *
   * @param conf Configuration
   * @throws IOException
   */
  public static void setupJob(Configuration conf) throws IOException {
    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.setTableDesc(getHiveTableName());
    Map<String, String> partitionValues = ImmutableMap.of("ds", "2013-04-01");
    outputDesc.setPartitionValues(partitionValues);
    LOG.info("Writing to {}", outputDesc);
    try {
      HiveApiOutputFormat.initProfile(conf, outputDesc, SampleOutputFormat.SAMPLE_PROFILE_ID);
    } catch (IOException e) {
      LOG.error("Failed to initialize profile {}", outputDesc);
View Full Code Here

Examples of com.facebook.hiveio.output.HiveOutputDescription

53
54
55
56
57
58
59
60
61
62
63
  }

  @Test
  public void testCheck() throws Exception {
    VertexToHive vertexToHive = new HiveOutputIntIntVertex();
    HiveOutputDescription outputDesc = new HiveOutputDescription();
    HiveTableSchema schema = TestSchema.builder()
        .addColumn("foo", HiveType.LONG)
        .addColumn("bar", HiveType.LONG)
        .build();
    vertexToHive.checkOutput(outputDesc, schema, newWritableRecord(schema));
View Full Code Here

Examples of com.facebook.hiveio.output.HiveOutputDescription

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
  public static final List<MapWritable> mapperData2 = ImmutableList.of(row3, row4);

  private HiveTools() {}

  public static void setupJob(Configuration conf) throws IOException {
    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.setDbName("default");
    outputDesc.setTableName("hive_io_test");
    Map<String, String> partitionValues = ImmutableMap.of("ds", "2013-04-01");
    outputDesc.setPartitionValues(partitionValues);
    LOG.info("Writing to {}", outputDesc);
    try {
      HiveApiOutputFormat.initProfile(conf, outputDesc, SampleOutputFormat.SAMPLE_PROFILE_ID);
    } catch (TException e) {
      LOG.error("Failed to initialize profile {}", outputDesc);
View Full Code Here

Examples of com.facebook.hiveio.output.HiveOutputDescription

114
115
116
117
118
119
120
121
122
123
124
125
126
127
   *
   * @param conf Configuration
   * @throws IOException
   */
  public static void setupJob(Configuration conf) throws IOException {
    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.setTableDesc(getHiveTableName());
    Map<String, String> partitionValues = ImmutableMap.of("ds", "2013-04-01");
    outputDesc.setPartitionValues(partitionValues);
    LOG.info("Writing to {}", outputDesc);
    try {
      HiveApiOutputFormat.initProfile(conf, outputDesc, SampleOutputFormat.SAMPLE_PROFILE_ID);
    } catch (TException e) {
      LOG.error("Failed to initialize profile {}", outputDesc);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.