Examples of ImmutableClassesGiraphConfiguration

org.apache.giraph.conf.ImmutableClassesGiraphConfiguration
The classes set here are immutable, the remaining configuration is mutable. Classes are immutable and final to provide the best performance for instantiation. Everything is thread-safe. @param < I> Vertex id @param < V> Vertex data @param < E> Edge data @param < M> Message data

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

   * Set the concrete, user-defined choices about generic methods
   * (validated earlier in GiraphRunner) into the Configuration.
   * @param conf the Configuration object for this job run.
   */
  public void determineClassTypes(Configuration conf) {
    ImmutableClassesGiraphConfiguration giraphConf =
        new ImmutableClassesGiraphConfiguration(conf);
    Class<? extends Vertex<I, V, E, M>> vertexClass =
        giraphConf.getVertexClass();
    List<Class<?>> classList = ReflectionUtils.<Vertex>getTypeArguments(
        Vertex.class, vertexClass);
    Type vertexIndexType = classList.get(0);
    Type vertexValueType = classList.get(1);
    Type edgeValueType = classList.get(2);

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

   *             from and update with any changes to the returned configuration
   * @return ImmutableClassesGiraphConfiguration
   */
  private ImmutableClassesGiraphConfiguration createGiraphConf(
      final Configuration conf) {
    return new ImmutableClassesGiraphConfiguration(conf) {
      @Override
      public void set(String name, String value) {
        super.set(name, value);
        conf.set(name, value);
      }

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

   * @return ImmutableClassesGiraphConfiguration
   */
  private ImmutableClassesGiraphConfiguration createGiraphConf(
      Configuration conf,
      final InputFormatDescription inputFormatDescription) {
    return new ImmutableClassesGiraphConfiguration(conf) {
      @Override
      public void set(String name, String value) {
        super.set(name, value);
        inputFormatDescription.addParameter(name, value);
      }

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

      HiveTableSchema schema, HiveWritableRecord emptyRecord) { }


  @Override
  public void initialize() {
    HiveTableSchema schema = getTableSchema();
    ImmutableClassesGiraphConfiguration conf = getConf();


    vertexIdWriter = HiveJythonUtils.newValueWriter(schema,
        VERTEX_ID_COLUMN, conf, GraphType.VERTEX_ID,
        GiraphHiveConstants.VERTEX_ID_WRITER_JYTHON_NAME);
    vertexValueWriter = HiveJythonUtils.newValueWriter(schema,

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

  @Override
  public void initializeRecords(Iterator<HiveReadableRecord> records) {
    super.initializeRecords(records);


    HiveTableSchema schema = getTableSchema();
    ImmutableClassesGiraphConfiguration conf = getConf();


    sourceIdReader = HiveJythonUtils.<I>newValueReader(schema,
        EDGE_SOURCE_ID_COLUMN, conf, GraphType.VERTEX_ID,
        GiraphHiveConstants.VERTEX_ID_READER_JYTHON_NAME);
    targetIdReader = HiveJythonUtils.<I>newValueReader(schema,

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

    private I vertexId;


    @Override
    public void initialize(TaskAttemptContext context)
      throws IOException, InterruptedException {
      ImmutableClassesGiraphConfiguration conf = getConf();


      vlabel = GIRAPH_REXSTER_VLABEL.get(conf);
      txsize = GIRAPH_REXSTER_OUTPUT_V_TXSIZE.get(conf);
      startConnection();


      /* set the barrier */
      zk = new ZooKeeperExt(conf.getZookeeperList(),
          conf.getZooKeeperSessionTimeout(), conf.getZookeeperOpsMaxAttempts(),
          conf.getZookeeperOpsRetryWaitMsecs(), this, context);
    }

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

    return values;
  }


  public static void commitJob(GiraphConfiguration conf)
    throws IOException, InterruptedException {
    ImmutableClassesGiraphConfiguration iconf = new ImmutableClassesGiraphConfiguration(conf);
    WrappedVertexOutputFormat outputFormat = iconf.createWrappedVertexOutputFormat();
    JobConf jobConf = new JobConf(conf);
    TaskAttemptContext
        taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID());
    OutputCommitter outputCommitter = outputFormat.getOutputCommitter(
        taskContext);

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

  private static Logger LOG = Logger.getLogger(BspOutputFormat.class);


  @Override
  public void checkOutputSpecs(JobContext context)
    throws IOException, InterruptedException {
    ImmutableClassesGiraphConfiguration conf =
        new ImmutableClassesGiraphConfiguration(context.getConfiguration());
    if (!conf.hasVertexOutputFormat() && !conf.hasEdgeOutputFormat()) {
      LOG.warn("checkOutputSpecs: ImmutableOutputCommiter" +
          " will not check anything");
      return;
    }


    if (conf.hasVertexOutputFormat()) {
      conf.createWrappedVertexOutputFormat().checkOutputSpecs(context);
    }
    if (conf.hasEdgeOutputFormat()) {
      conf.createWrappedEdgeOutputFormat().checkOutputSpecs(context);
    }
  }

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

   *         valid input splits.
   */
  private int createInputSplits(GiraphInputFormat inputFormat,
                                InputSplitPaths inputSplitPaths,
                                String inputSplitType) {
    ImmutableClassesGiraphConfiguration conf = getConfiguration();
    String logPrefix = "create" + inputSplitType + "InputSplits";
    // Only the 'master' should be doing this.  Wait until the number of
    // processes that have reported health exceeds the minimum percentage.
    // If the minimum percentage is not met, fail the job.  Otherwise
    // generate the input splits
    String inputSplitsPath = inputSplitPaths.getPath();
    try {
      if (getZkExt().exists(inputSplitsPath, false) != null) {
        LOG.info(inputSplitsPath + " already exists, no need to create");
        return Integer.parseInt(
            new String(getZkExt().getData(inputSplitsPath, false, null),
                Charset.defaultCharset()));
      }
    } catch (KeeperException.NoNodeException e) {
      if (LOG.isInfoEnabled()) {
        LOG.info(logPrefix + ": Need to create the input splits at " +
            inputSplitsPath);
      }
    } catch (KeeperException e) {
      throw new IllegalStateException(logPrefix + ": KeeperException", e);
    } catch (InterruptedException e) {
      throw new IllegalStateException(logPrefix + ": InterruptedException", e);
    }


    // When creating znodes, in case the master has already run, resume
    // where it left off.
    List<WorkerInfo> healthyWorkerInfoList = checkWorkers();
    if (healthyWorkerInfoList == null) {
      setJobStateFailed("Not enough healthy workers to create input splits");
      return -1;
    }


    // Create at least as many splits as the total number of input threads.
    int minSplitCountHint = healthyWorkerInfoList.size() *
        conf.getNumInputSplitsThreads();


    // Note that the input splits may only be a sample if
    // INPUT_SPLIT_SAMPLE_PERCENT is set to something other than 100
    List<InputSplit> splitList = generateInputSplits(inputFormat,
        minSplitCountHint, inputSplitType);


    if (splitList.isEmpty()) {
      LOG.fatal(logPrefix + ": Failing job due to 0 input splits, " +
          "check input of " + inputFormat.getClass().getName() + "!");
      getContext().setStatus("Failing job due to 0 input splits, " +
          "check input of " + inputFormat.getClass().getName() + "!");
      setJobStateFailed("Please check your input tables - partitions which " +
          "you specified are missing. Failing the job!!!");
    }
    if (minSplitCountHint > splitList.size()) {
      LOG.warn(logPrefix + ": Number of inputSplits=" +
          splitList.size() + " < " +
          minSplitCountHint +
          "=total number of input threads, " +
          "some threads will be not used");
    }


    // Write input splits to zookeeper in parallel
    int inputSplitThreadCount = conf.getInt(NUM_MASTER_ZK_INPUT_SPLIT_THREADS,
        DEFAULT_INPUT_SPLIT_THREAD_COUNT);
    if (LOG.isInfoEnabled()) {
      LOG.info(logPrefix + ": Starting to write input split data " +
          "to zookeeper with " + inputSplitThreadCount + " threads");
    }

View Full Code Here

Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

   *
   * @param superstep Superstep to aggregate on
   * @return Global statistics aggregated on all worker statistics
   */
  private GlobalStats aggregateWorkerStats(long superstep) {
    ImmutableClassesGiraphConfiguration conf = getConfiguration();


    Class<? extends PartitionStats> partitionStatsClass =
        masterGraphPartitioner.createPartitionStats().getClass();
    GlobalStats globalStats = new GlobalStats();
    // Get the stats from the all the worker selected nodes
    String workerFinishedPath =
        getWorkerFinishedPath(getApplicationAttempt(), superstep);
    List<String> workerFinishedPathList = null;
    try {
      workerFinishedPathList =
          getZkExt().getChildrenExt(
              workerFinishedPath, false, false, true);
    } catch (KeeperException e) {
      throw new IllegalStateException(
          "aggregateWorkerStats: KeeperException", e);
    } catch (InterruptedException e) {
      throw new IllegalStateException(
          "aggregateWorkerStats: InterruptedException", e);
    }


    AggregatedMetrics aggregatedMetrics = new AggregatedMetrics();


    allPartitionStatsList.clear();
    for (String finishedPath : workerFinishedPathList) {
      String hostnamePartitionId = FilenameUtils.getName(finishedPath);
      JSONObject workerFinishedInfoObj = null;
      try {
        byte [] zkData =
            getZkExt().getData(finishedPath, false, null);
        workerFinishedInfoObj = new JSONObject(new String(zkData,
            Charset.defaultCharset()));
        List<PartitionStats> statsList =
            WritableUtils.readListFieldsFromByteArray(
                Base64.decode(workerFinishedInfoObj.getString(
                    JSONOBJ_PARTITION_STATS_KEY)),
                    partitionStatsClass,
                    conf);
        for (PartitionStats partitionStats : statsList) {
          globalStats.addPartitionStats(partitionStats);
          allPartitionStatsList.add(partitionStats);
        }
        globalStats.addMessageCount(
            workerFinishedInfoObj.getLong(
                JSONOBJ_NUM_MESSAGES_KEY));
        globalStats.addMessageBytesCount(
          workerFinishedInfoObj.getLong(
              JSONOBJ_NUM_MESSAGE_BYTES_KEY));
        if (conf.metricsEnabled() &&
            workerFinishedInfoObj.has(JSONOBJ_METRICS_KEY)) {
          WorkerSuperstepMetrics workerMetrics = new WorkerSuperstepMetrics();
          WritableUtils.readFieldsFromByteArray(
              Base64.decode(
                  workerFinishedInfoObj.getString(
                      JSONOBJ_METRICS_KEY)),
              workerMetrics);
          aggregatedMetrics.add(workerMetrics, hostnamePartitionId);
        }
      } catch (JSONException e) {
        throw new IllegalStateException(
            "aggregateWorkerStats: JSONException", e);
      } catch (KeeperException e) {
        throw new IllegalStateException(
            "aggregateWorkerStats: KeeperException", e);
      } catch (InterruptedException e) {
        throw new IllegalStateException(
            "aggregateWorkerStats: InterruptedException", e);
      } catch (IOException e) {
        throw new IllegalStateException(
            "aggregateWorkerStats: IOException", e);
      }
    }


    if (conf.metricsEnabled()) {
      if (GiraphConstants.METRICS_DIRECTORY.isDefaultValue(conf)) {
        aggregatedMetrics.print(superstep, System.err);
      } else {
        printAggregatedMetricsToHDFS(superstep, aggregatedMetrics);
      }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.