Examples of ArrayWritable

org.apache.hadoop.io.ArrayWritable
A Writable for arrays containing instances of a class.

Examples of org.apache.hadoop.io.ArrayWritable


    @Test
    public void testMap() {
        LinkedMapWritable map = new LinkedMapWritable();
        map.put(new Text("key"), new IntWritable(1));
        map.put(new BooleanWritable(Boolean.TRUE), new ArrayWritable(new String[] { "one", "two" }));
        writableTypeToJson(map);
    }

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable

        switch (type.getCategory()) {
        case LIST: {// or ARRAY
            ListTypeInfo listType = (ListTypeInfo) type;
            TypeInfo listElementType = listType.getListElementTypeInfo();


            ArrayWritable aw = (ArrayWritable) data;


            List<Object> list = new ArrayList<Object>();
            for (Writable writable : aw.get()) {
                list.add(hiveFromWritable(listElementType, writable, alias, IS_ES_10));
            }


            return list;
        }

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable

      FSImageSerialization.writeString(path, out);
      FSImageSerialization.writeShortAsString(replication, out);
      FSImageSerialization.writeLongAsString(mtime, out);
      FSImageSerialization.writeLongAsString(atime, out);
      FSImageSerialization.writeLongAsString(blockSize, out);
      new ArrayWritable(Block.class, blocks).write(out);
      permissions.write(out);


      if (this.opCode == OP_ADD) {
        FSImageSerialization.writeString(clientName,out);
        FSImageSerialization.writeString(clientMachine,out);

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable


  private static void checkSuperstepMsgCount(PeerSyncClient syncClient,
      @SuppressWarnings("rawtypes")
      BSPPeer bspTask, BSPJob job, long step, long count) {


    ArrayWritable writableVal = new ArrayWritable(LongWritable.class);


    boolean result = syncClient.getInformation(
        syncClient.constructKey(job.getJobID(), "checkpoint",
            "" + bspTask.getPeerIndex()), writableVal);


    assertTrue(result);


    LongWritable superstepNo = (LongWritable) writableVal.get()[0];
    LongWritable msgCount = (LongWritable) writableVal.get()[1];


    assertEquals(step, superstepNo.get());
    assertEquals(count, msgCount.get());
  }

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable

    int port = BSPNetUtils.getFreePort(12502);
    LOG.info("Got port = " + port);


    boolean result = syncClient
        .getInformation(syncClient.constructKey(job.getJobID(), "checkpoint",
            "" + bspTask.getPeerIndex()), new ArrayWritable(LongWritable.class));


    assertFalse(result);


    bspTask.sync();
    // Superstep 1

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable

    String writeKey = "job_checkpttest_0001/checkpoint/1/";


    Writable[] writableArr = new Writable[2];
    writableArr[0] = new LongWritable(3L);
    writableArr[1] = new LongWritable(5L);
    ArrayWritable arrWritable = new ArrayWritable(LongWritable.class);
    arrWritable.set(writableArr);
    syncClient.storeInformation(writeKey, arrWritable, true, null);


    String writePath = "checkpoint/job_checkpttest_0001/3/1";
    FSDataOutputStream out = dfs.create(new Path(writePath));
    for (int i = 0; i < 5; ++i) {

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable

    return readText(null);
  }


  public ArrayWritable readArray(ArrayWritable aw) throws IOException {
    if (aw == null) {
      aw = new ArrayWritable(TypedBytesWritable.class);
    } else if (!aw.getValueClass().equals(TypedBytesWritable.class)) {
      throw new RuntimeException("value class has to be TypedBytesWritable");
    }
    int length = in.readVectorHeader();
    Writable[] writables = new Writable[length];

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable

    }


    @Override
    public boolean storeInformation(String key, Writable value,
        boolean permanent, SyncEventListener listener) {
      ArrayWritable writables = (ArrayWritable) value;
      long step = ((LongWritable) writables.get()[0]).get();
      long count = ((LongWritable) writables.get()[1]).get();


      LOG.info("SyncClient Storing value step = " + step + " count = " + count
          + " for key " + key);
      valueMap.put(key, value);
      return true;

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable

  public TextArrayWritable() {
    this(Collections.EMPTY_LIST);
  }


  public TextArrayWritable(List<Text> texts) {
    array = new ArrayWritable(Text.class, texts.toArray(new Writable[texts.size()]));
  }

View Full Code Here

Examples of org.apache.hadoop.io.ArrayWritable

          }
          numEdits++;
          switch (opcode) {
          case OP_ADD: {
            UTF8 name = new UTF8();
            ArrayWritable aw = null;
            Writable writables[];
            // version 0 does not support per file replication
            if (logVersion >= 0)
              name.readFields(in);  // read name only
            else {  // other versions do
              // get name and replication
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get(); 
              if (logVersion >= -4 && writables.length != 2 ||
                  logVersion < -4 && writables.length != 3) {
                  throw new IOException("Incorrect data fortmat. " 
                                        + "Name & replication pair expected");
              }
              name = (UTF8) writables[0];
              replication = Short.parseShort(
                                             ((UTF8)writables[1]).toString());
              replication = adjustReplication(replication);
              if (logVersion < -4) {
                mtime = Long.parseLong(((UTF8)writables[2]).toString());
              }
            }
            // get blocks
            aw = new ArrayWritable(Block.class);
            aw.readFields(in);
            writables = aw.get();
            Block blocks[] = new Block[writables.length];
            System.arraycopy(writables, 0, blocks, 0, blocks.length);
            // add to the file tree
            fsDir.unprotectedAddFile(name.toString(), blocks, replication, mtime);
            break;
          }
          case OP_SET_REPLICATION: {
            UTF8 src = new UTF8();
            UTF8 repl = new UTF8();
            src.readFields(in);
            repl.readFields(in);
            replication = adjustReplication(fromLogReplication(repl));
            fsDir.unprotectedSetReplication(src.toString(), 
                                            replication,
                                            null);
            break;
          } 
          case OP_RENAME: {
            UTF8 src = null;
            UTF8 dst = null;
            if (logVersion >= -4) {
              src = new UTF8();
              dst = new UTF8();
              src.readFields(in);
              dst.readFields(in);
            } else {
              ArrayWritable aw = null;
              Writable writables[];
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get(); 
              if (writables.length != 3) {
                throw new IOException("Incorrect data fortmat. " 
                                      + "Mkdir operation.");
              }
              src = (UTF8) writables[0];
              dst = (UTF8) writables[1];
              timestamp = Long.parseLong(((UTF8)writables[2]).toString());
            }
            fsDir.unprotectedRenameTo(src.toString(), dst.toString(), timestamp);
            break;
          }
          case OP_DELETE: {
            UTF8 src = null;
            if (logVersion >= -4) {
              src = new UTF8();
              src.readFields(in);
            } else {
              ArrayWritable aw = null;
              Writable writables[];
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get(); 
              if (writables.length != 2) {
                throw new IOException("Incorrect data fortmat. " 
                                      + "delete operation.");
              }
              src = (UTF8) writables[0];
              timestamp = Long.parseLong(((UTF8)writables[1]).toString());
            }
            fsDir.unprotectedDelete(src.toString(), timestamp);
            break;
          }
          case OP_MKDIR: {
            UTF8 src = null;
            if (logVersion >= -4) {
              src = new UTF8();
              src.readFields(in);
            } else {
              ArrayWritable aw = null;
              Writable writables[];
              aw = new ArrayWritable(UTF8.class);
              aw.readFields(in);
              writables = aw.get(); 
              if (writables.length != 2) {
                throw new IOException("Incorrect data fortmat. " 
                                      + "Mkdir operation.");
              }
              src = (UTF8) writables[0];

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.