Examples of PartitionStats


Examples of org.apache.giraph.graph.partition.PartitionStats

            context.progress();

            partitionStatsList.clear();
            for (Partition<I, V, E, M> partition :
                    serviceWorker.getPartitionMap().values()) {
                PartitionStats partitionStats =
                    new PartitionStats(partition.getPartitionId(), 0, 0, 0);
                for (BasicVertex<I, V, E, M> basicVertex :
                        partition.getVertices()) {
                    // Make sure every vertex has the current
                    // graphState before computing
                    basicVertex.setGraphState(graphState);
                    if (basicVertex.isHalted()
                            && !Iterables.isEmpty(basicVertex.getMessages())) {
                        basicVertex.halt = false;
                    }
                    if (!basicVertex.isHalted()) {
                        Iterator<M> vertexMsgIt =
                            basicVertex.getMessages().iterator();
                        context.progress();
                        basicVertex.compute(vertexMsgIt);
                        basicVertex.releaseResources();
                    }
                    if (basicVertex.isHalted()) {
                        partitionStats.incrFinishedVertexCount();
                    }
                    partitionStats.incrVertexCount();
                    partitionStats.addEdgeCount(basicVertex.getNumOutEdges());
                }
                partitionStatsList.add(partitionStats);
            }
        } while (!serviceWorker.finishSuperstep(partitionStatsList));
        if (LOG.isInfoEnabled()) {
View Full Code Here

Examples of org.apache.giraph.graph.partition.PartitionStats

        // Generate the partition stats for the input superstep and process
        // if necessary
        List<PartitionStats> partitionStatsList =
            new ArrayList<PartitionStats>();
        for (Partition<I, V, E, M> partition : getPartitionMap().values()) {
            PartitionStats partitionStats =
                new PartitionStats(partition.getPartitionId(),
                                   partition.getVertices().size(),
                                   0,
                                   partition.getEdgeCount());
            partitionStatsList.add(partitionStats);
        }
View Full Code Here

Examples of org.apache.giraph.partition.PartitionStats

      }

      Partition<I, V, E, M> partition =
          serviceWorker.getPartitionStore().getPartition(partitionId);
      try {
        PartitionStats partitionStats = computePartition(partition);
        partitionStatsList.add(partitionStats);
        long partitionMsgs = workerClientRequestProcessor.resetMessageCount();
        partitionStats.addMessagesSentCount(partitionMsgs);
        messagesSentCounter.inc(partitionMsgs);
        timedLogger.info("call: Completed " +
            partitionStatsList.size() + " partitions, " +
            partitionIdQueue.size() + " remaining " +
            MemoryUtils.getRuntimeMemoryStats());
View Full Code Here

Examples of org.apache.giraph.partition.PartitionStats

   * @param partition Partition to compute
   * @return Partition stats for this computed partition
   */
  private PartitionStats computePartition(Partition<I, V, E, M> partition)
    throws IOException, InterruptedException {
    PartitionStats partitionStats =
        new PartitionStats(partition.getId(), 0, 0, 0, 0);
    // Make sure this is thread-safe across runs
    synchronized (partition) {
      // Prepare Partition context
      WorkerContext workerContext =
          graphState.getGraphTaskManager().getWorkerContext();
      PartitionContext partitionContext = partition.getPartitionContext();
      synchronized (workerContext) {
        partitionContext.preSuperstep(workerContext);
      }
      graphState.setPartitionContext(partition.getPartitionContext());

      for (Vertex<I, V, E, M> vertex : partition) {
        // Make sure every vertex has this thread's
        // graphState before computing
        vertex.setGraphState(graphState);
        Iterable<M> messages = messageStore.getVertexMessages(vertex.getId());
        if (vertex.isHalted() && !Iterables.isEmpty(messages)) {
          vertex.wakeUp();
        }
        if (!vertex.isHalted()) {
          context.progress();
          TimerContext computeOneTimerContext = computeOneTimer.time();
          try {
            vertex.compute(messages);
          } finally {
            computeOneTimerContext.stop();
          }
          // Need to unwrap the mutated edges (possibly)
          vertex.unwrapMutableEdges();
          // Write vertex to superstep output (no-op if it is not used)
          vertexWriter.writeVertex(vertex);
          // Need to save the vertex changes (possibly)
          partition.saveVertex(vertex);
        }
        if (vertex.isHalted()) {
          partitionStats.incrFinishedVertexCount();
        }
        // Remove the messages now that the vertex has finished computation
        messageStore.clearVertexMessages(vertex.getId());

        // Add statistics for this vertex
        partitionStats.incrVertexCount();
        partitionStats.addEdgeCount(vertex.getNumEdges());
      }

      messageStore.clearPartition(partition.getId());

      synchronized (workerContext) {
View Full Code Here

Examples of org.apache.giraph.partition.PartitionStats

    List<PartitionStats> partitionStatsList =
        new ArrayList<PartitionStats>();
    for (Integer partitionId : getPartitionStore().getPartitionIds()) {
      Partition<I, V, E, M> partition =
          getPartitionStore().getPartition(partitionId);
      PartitionStats partitionStats =
          new PartitionStats(partition.getId(),
              partition.getVertexCount(),
              0,
              partition.getEdgeCount(),
              0);
      partitionStatsList.add(partitionStats);
View Full Code Here

Examples of org.apache.giraph.partition.PartitionStats

      computation.initialize(graphState, workerClientRequestProcessor,
          serviceWorker.getGraphTaskManager(), aggregatorUsage, workerContext);
      computation.preSuperstep();

      try {
        PartitionStats partitionStats =
            computePartition(computation, partition);
        partitionStatsList.add(partitionStats);
        long partitionMsgs = workerClientRequestProcessor.resetMessageCount();
        partitionStats.addMessagesSentCount(partitionMsgs);
        messagesSentCounter.inc(partitionMsgs);
        long partitionMsgBytes =
          workerClientRequestProcessor.resetMessageBytesCount();
        partitionStats.addMessageBytesSentCount(partitionMsgBytes);
        messageBytesSentCounter.inc(partitionMsgBytes);
        timedLogger.info("call: Completed " +
            partitionStatsList.size() + " partitions, " +
            partitionIdQueue.size() + " remaining " +
            MemoryUtils.getRuntimeMemoryStats());
View Full Code Here

Examples of org.apache.giraph.partition.PartitionStats

   * @return Partition stats for this computed partition
   */
  private PartitionStats computePartition(
      Computation<I, V, E, M1, M2> computation,
      Partition<I, V, E> partition) throws IOException, InterruptedException {
    PartitionStats partitionStats =
        new PartitionStats(partition.getId(), 0, 0, 0, 0, 0);
    long verticesComputedProgress = 0;
    // Make sure this is thread-safe across runs
    synchronized (partition) {
      for (Vertex<I, V, E> vertex : partition) {
        Iterable<M1> messages = messageStore.getVertexMessages(vertex.getId());
        if (vertex.isHalted() && !Iterables.isEmpty(messages)) {
          vertex.wakeUp();
        }
        if (!vertex.isHalted()) {
          context.progress();
          computation.compute(vertex, messages);
          // Need to unwrap the mutated edges (possibly)
          vertex.unwrapMutableEdges();
          //Compact edges representation if possible
          if (vertex instanceof Trimmable) {
            ((Trimmable) vertex).trim();
          }
          // Write vertex to superstep output (no-op if it is not used)
          vertexWriter.writeVertex(vertex);
          // Need to save the vertex changes (possibly)
          partition.saveVertex(vertex);
        }
        if (vertex.isHalted()) {
          partitionStats.incrFinishedVertexCount();
        }
        // Remove the messages now that the vertex has finished computation
        messageStore.clearVertexMessages(vertex.getId());

        // Add statistics for this vertex
        partitionStats.incrVertexCount();
        partitionStats.addEdgeCount(vertex.getNumEdges());

        verticesComputedProgress++;
        if (verticesComputedProgress == VERTICES_TO_UPDATE_PROGRESS) {
          WorkerProgress.get().addVerticesComputed(verticesComputedProgress);
          verticesComputedProgress = 0;
View Full Code Here

Examples of org.apache.giraph.partition.PartitionStats

    List<PartitionStats> partitionStatsList =
        new ArrayList<PartitionStats>();
    for (Integer partitionId : getPartitionStore().getPartitionIds()) {
      Partition<I, V, E> partition =
          getPartitionStore().getOrCreatePartition(partitionId);
      PartitionStats partitionStats =
          new PartitionStats(partition.getId(),
              partition.getVertexCount(),
              0,
              partition.getEdgeCount(),
              0, 0);
      partitionStatsList.add(partitionStats);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.