Examples of HiveInputDescription

Examples of com.facebook.giraph.hive.input.HiveInputDescription

103
104
105
106
107
108
109
110
111
  private Configuration conf;

  /** Create a new runner */
  public HiveGiraphRunner() {
    conf = new HiveConf(getClass());
    hiveVertexInputDescription = new HiveInputDescription();
    hiveEdgeInputDescription = new HiveInputDescription();
    hiveOutputDescription = new HiveOutputDescription();
  }
View Full Code Here

Examples of com.facebook.giraph.hive.input.HiveInputDescription

99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
    }

    ThriftHiveMetastore.Iface client = HiveMetastores.create(metastoreHostPort.host,
        metastoreHostPort.port);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setDbName(opts.database);
    inputDesc.setTableName(opts.table);
    inputDesc.setPartitionFilter(opts.partitionFilter);
    if (opts.requestNumSplits == 0) {
      opts.requestNumSplits = opts.threads * opts.requestSplitsPerThread;
    }
    inputDesc.setNumSplits(opts.requestNumSplits);

    HiveStats hiveStats = HiveUtils.statsOf(client, inputDesc);
    System.err.println(hiveStats);

    HiveConf hiveConf = new HiveConf(Tailer.class);
View Full Code Here

Examples of com.facebook.giraph.hive.input.HiveInputDescription

75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    }

    ThriftHiveMetastore.Iface client = HiveMetastores.create(metastoreHostPort.host,
        metastoreHostPort.port);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setDbName(opts.database);
    inputDesc.setTableName(opts.table);
    inputDesc.setPartitionFilter(opts.partitionFilter);
    if (opts.requestNumSplits == 0) {
      opts.requestNumSplits = opts.threads * opts.requestSplitsPerThread;
    }
    inputDesc.setNumSplits(opts.requestNumSplits);

    HiveStats hiveStats = HiveUtils.statsOf(client, inputDesc);
    LOG.info(hiveStats);

    HiveConf hiveConf = new HiveConf(Tailer.class);
View Full Code Here

Examples of com.facebook.giraph.hive.input.HiveInputDescription

101
102
103
104
105
106
107
108
109
110
111
112
113
114
    return Optional.of(parsedArgs);
  }

  private static void run(BenchmarkArgs parsedArgs)
      throws TTransportException, IOException, InterruptedException {
    HiveInputDescription input = new HiveInputDescription();
    input.setDbName(parsedArgs.getDatabase());
    input.setTableName(parsedArgs.getTable());
    input.setPartitionFilter(parsedArgs.getPartitionFilter());

    HiveConf hiveConf = new HiveConf(InputBenchmark.class);
    ThriftHiveMetastore.Iface client = HiveMetastores.create(parsedArgs.getHiveHost(), parsedArgs.getHivePort());

    System.err.println("Initialize profile with input data");
View Full Code Here

Examples of com.facebook.giraph.hive.input.HiveInputDescription

61
62
63
64
65
66
67
68
69
70
71
   * @throws IOException I/O errors
   * @throws InterruptedException thread errors
   */
  public static void main(String[] args)
    throws TTransportException, IOException, InterruptedException {
    HiveInputDescription input = new HiveInputDescription();
    BenchmarkArgs parsedArgs = handleCommandLine(args, input);
    if (parsedArgs == null) {
      return;
    }

View Full Code Here

Examples of com.facebook.giraph.hive.input.HiveInputDescription

102
103
104
105
106
107
108
109
110
111
112
113
114
115
    return Optional.of(parsedArgs);
  }

  private static void run(BenchmarkArgs parsedArgs)
      throws TTransportException, IOException, InterruptedException {
    HiveInputDescription input = new HiveInputDescription();
    input.setDbName(parsedArgs.getDatabase());
    input.setTableName(parsedArgs.getTable());
    input.setPartitionFilter(parsedArgs.getPartitionFilter());

    HiveConf hiveConf = new HiveConf(InputBenchmark.class);
    ThriftHiveMetastore.Iface client = HiveMetastores
        .create(parsedArgs.getHiveHost(), parsedArgs.getHivePort());
View Full Code Here

Examples of com.facebook.hiveio.input.HiveInputDescription

55
56
57
58
59
60
61
62
63
64
65
66
67
68
    HadoopNative.requireHadoopNative();

    Timer allTime = Metrics.newTimer(InputBenchmark.class, "all-time", MILLISECONDS, MILLISECONDS);
    TimerContext allTimerContext = allTime.time();

    HiveInputDescription input = new HiveInputDescription();
    input.setDbName(args.tableOpts.database);
    input.setTableName(args.tableOpts.table);
    input.setPartitionFilter(args.tableOpts.partitionFilter);

    HiveConf hiveConf = new HiveConf(InputBenchmark.class);
    ThriftHiveMetastore.Iface client = HiveMetastores.create(args.metastoreOpts.hiveHost, args.metastoreOpts.hivePort);

    System.err.println("Initialize profile with input data");
View Full Code Here

Examples of com.facebook.hiveio.input.HiveInputDescription

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
    LOG.info("Creating Hive client for Metastore at {}", metastoreHostPort);
    ThriftHiveMetastore.Iface client = HiveMetastores.create(metastoreHostPort.host,
        metastoreHostPort.port);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setDbName(opts.tableOpts.database);
    inputDesc.setTableName(opts.tableOpts.table);
    inputDesc.setPartitionFilter(opts.tableOpts.partitionFilter);
    if (opts.requestNumSplits == 0) {
      opts.requestNumSplits = opts.threads * opts.requestSplitsPerThread;
    }
    inputDesc.setNumSplits(opts.requestNumSplits);

    HiveStats hiveStats = HiveUtils.statsOf(client, inputDesc);
    LOG.info("{}", hiveStats);

    HiveConf hiveConf = new HiveConf(Tailer.class);
View Full Code Here

Examples of com.facebook.hiveio.input.HiveInputDescription

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    HadoopNative.requireHadoopNative();

    Timer allTime = Metrics.newTimer(InputBenchmark.class, "all-time", MILLISECONDS, MILLISECONDS);
    TimerContext allTimerContext = allTime.time();

    HiveInputDescription input = new HiveInputDescription();
    input.setDbName(args.tableOpts.database);
    input.setTableName(args.tableOpts.table);
    input.setPartitionFilter(args.tableOpts.partitionFilter);
    input.getMetastoreDesc().setHost(args.metastoreOpts.host);
    input.getMetastoreDesc().setPort(args.metastoreOpts.port);

    HiveConf hiveConf = HiveUtils.newHiveConf(InputBenchmark.class);

    System.err.println("Initialize profile with input data");
    HiveApiInputFormat.setProfileInputDesc(hiveConf, input, DEFAULT_PROFILE_ID);
View Full Code Here

Examples of com.facebook.hiveio.input.HiveInputDescription

88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    LOG.info("Creating Hive client for Metastore at {}", metastoreHostPort);
    ThriftHiveMetastore.Iface client = HiveMetastores.create(
        metastoreHostPort.host, metastoreHostPort.port);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setDbName(args.inputTable.database);
    inputDesc.setTableName(args.inputTable.table);
    inputDesc.setPartitionFilter(args.inputTable.partitionFilter);
    if (args.requestNumSplits == 0) {
      args.requestNumSplits = args.multiThread.threads * args.requestSplitsPerThread;
    }
    inputDesc.setNumSplits(args.requestNumSplits);
    inputDesc.getMetastoreDesc().setHost(metastoreHostPort.host);
    inputDesc.getMetastoreDesc().setPort(metastoreHostPort.port);

    HiveStats hiveStats = HiveUtils.statsOf(client, inputDesc);
    LOG.info("{}", hiveStats);

    HiveConf hiveConf = HiveUtils.newHiveConf(TailerCmd.class);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.