Package org.apache.giraph.utils

Examples of org.apache.giraph.utils.ExtendedDataOutput

201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
    ByteArrayOneMessageToManyIds<IntWritable, IntWritable>
        dataToSend = new ByteArrayOneMessageToManyIds<>(new
        TestMessageValueFactory<>(IntWritable.class));
    dataToSend.setConf(conf);
    dataToSend.initialize();
    ExtendedDataOutput output = conf.createExtendedDataOutput();
    for (int i = 1; i <= 7; ++i) {
      IntWritable vertexId = new IntWritable(i);
      vertexId.write(output);
    }
    dataToSend.add(output.getByteArray(), output.getPos(), 7, new IntWritable(1));

    // Send the request
    SendWorkerOneMessageToManyRequest<IntWritable, IntWritable> request =
      new SendWorkerOneMessageToManyRequest<>(dataToSend, conf);
    client.sendWritableRequest(workerInfo.getTaskId(), request);
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
      list = getOrCreateList(vertexIdMessageIterator);
      if (vertexIdMessageIterator.isNewMessage()) {
        IndexAndDataOut indexAndDataOut = bytesBuffer.getIndexAndDataOut();
        pointer = indexAndDataOut.getIndex();
        pointer <<= 32;
        ExtendedDataOutput dataOutput = indexAndDataOut.getDataOutput();
        pointer += dataOutput.getPos();
        msg.write(dataOutput);
      }
      synchronized (list) {
        list.add(pointer);
      }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
      public M next() {
        long pointer = iterator.next();
        try {
          int index = (int) (pointer >>> 32);
          int offset = (int) pointer;
          ExtendedDataOutput buffer = msgBuffer.getDataOutput(index);
          messageReader.initialize(buffer.getByteArray(), offset,
            buffer.getPos());
          reusableMsg.readFields(messageReader);
        } catch (IOException e) {
          throw new IllegalStateException("Got exception : " + e);
        }
        return reusableMsg;
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
   * @return Size of partitions for this worker
   */
  public int addVertex(PartitionOwner partitionOwner,
      Vertex<I, V, E> vertex) {
    // Get the data collection
    ExtendedDataOutput partitionData =
        getData(partitionOwner.getPartitionId());
    int taskId = partitionOwner.getWorkerInfo().getTaskId();
    int originalSize = 0;
    if (partitionData == null) {
      partitionData = getConf().createExtendedDataOutput(
          getInitialBufferSize(taskId));
      setData(partitionOwner.getPartitionId(), partitionData);
    } else {
      originalSize = partitionData.getPos();
    }
    try {
      WritableUtils.<I, V, E>writeVertexToDataOutput(
          partitionData, vertex, getConf());
    } catch (IOException e) {
      throw new IllegalStateException("addVertex: Failed to serialize", e);
    }

    // Update the size of cached, outgoing data per worker
    return incrDataSize(taskId, partitionData.getPos() - originalSize);
  }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
  /** Number of edges. */
  private int edgeCount;

  @Override
  public void initialize(Iterable<Edge<I, E>> edges) {
    ExtendedDataOutput extendedOutputStream =
        getConf().createExtendedDataOutput();
    for (Edge<I, E> edge : edges) {
      try {
        WritableUtils.writeEdge(extendedOutputStream, edge);
      } catch (IOException e) {
        throw new IllegalStateException("initialize: Failed to serialize " +
            edge);
      }
      ++edgeCount;
    }
    serializedEdges = extendedOutputStream.getByteArray();
    serializedEdgesBytesUsed = extendedOutputStream.getPos();
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    // since add() and iterator() work fine with a null buffer.
  }

  @Override
  public void add(Edge<I, E> edge) {
    ExtendedDataOutput extendedDataOutput =
        getConf().createExtendedDataOutput(
            serializedEdges, serializedEdgesBytesUsed);
    try {
      WritableUtils.writeEdge(extendedDataOutput, edge);
    } catch (IOException e) {
      throw new IllegalStateException("add: Failed to write to the new " +
          "byte array");
    }
    serializedEdges = extendedDataOutput.getByteArray();
    serializedEdgesBytesUsed = extendedDataOutput.getPos();
    ++edgeCount;
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
    int numPartitions = input.readInt();
    workerPartitions = new PairList<Integer, ExtendedDataOutput>();
    workerPartitions.initialize(numPartitions);
    while (numPartitions-- > 0) {
      final int partitionId = input.readInt();
      ExtendedDataOutput partitionData =
          WritableUtils.readExtendedDataOutput(input, getConf());
      workerPartitions.add(partitionId, partitionData);
    }
  }
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
      list = getList(iterator);
      if (iterator.isNewMessage()) {
        IndexAndDataOut indexAndDataOut = bytesBuffer.getIndexAndDataOut();
        pointer = indexAndDataOut.getIndex();
        pointer <<= 32;
        ExtendedDataOutput dataOutput = indexAndDataOut.getDataOutput();
        pointer += dataOutput.getPos();
        msg.write(dataOutput);
      }
      synchronized (list) { // TODO - any better way?
        list.add(pointer);
      }
View Full Code Here
TOP

Related Classes of org.apache.giraph.utils.ExtendedDataOutput

  • org.apache.giraph.comm.messages.ByteArrayMessagesPerVertexStore
  • org.apache.giraph.comm.messages.DiskBackedMessageStore
  • org.apache.giraph.comm.messages.DiskBackedMessageStore$TemporaryMessageStore
  • org.apache.giraph.comm.messages.PointerListMessagesIterable
  • org.apache.giraph.comm.messages.PointerListPerVertexStore
  • org.apache.giraph.comm.messages.primitives.long_id.LongPointerListMessageStore
  • org.apache.giraph.comm.requests.SendWorkerVerticesRequest
  • org.apache.giraph.comm.RequestTest
  • org.apache.giraph.comm.SendPartitionCache
  • org.apache.giraph.edge.ByteArrayEdges
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.