Package org.apache.hadoop.chukwa

Examples of org.apache.hadoop.chukwa.ChunkImpl


    try {
      while (!stopping) {
        byte[] data = new byte[BUFFER_SIZE];
        r.nextBytes(data);
        offset += data.length;
        ChunkImpl evt = new ChunkImpl(type, "random data source", offset, data,
            this);
        dest.add(evt);

      }
    } catch (InterruptedException ie) {
View Full Code Here


     
      Text t;
      BytesWritable bw;
     
      if(k instanceof ChukwaArchiveKey && v instanceof ChunkImpl) {
        ChunkImpl value = (ChunkImpl) v;
        Report xtrReport = Report.createFromString(new String(value.getData()));
      
        try {    //we do this to handle the case where not all input is x-trace
          bw = new BytesWritable(xtrReport.getMetadata().getTaskId().get());
        } catch(Exception e) {
          unparseableReport.increment(1);
          return;
        }
       
        //FIXME: can probably optimize the above lines by doing a search in the raw bytes
        t= new Text(value.getData());
      } else if(k instanceof ChukwaRecordKey && v instanceof ChukwaRecord){
        ChukwaRecord value = (ChukwaRecord) v;
        Report xtrReport = Report.createFromString(value.getValue(Record.bodyField));
        bw = new BytesWritable(xtrReport.getMetadata().getTaskId().get());
        //FIXME: can probably optimize the above lines by doing a search in the raw bytes
        t= new Text(value.getValue(Record.bodyField));
      } else {
        log.error("unexpected key/value types: "+ k.getClass().getCanonicalName()
            + " and " + v.getClass().getCanonicalName() );
        return;
      }
View Full Code Here

     SequenceFile.Writer seqFileWriter = SequenceFile.createWriter(conf, out,
         ChukwaArchiveKey.class, ChunkImpl.class,
         SequenceFile.CompressionType.NONE, null);
     RandSeqFileWriter rw = new RandSeqFileWriter();
     for (int i = 0; i < chunks; ++i) {
       ChunkImpl chunk = rw.getARandomChunk();
       ChukwaArchiveKey archiveKey = new ChukwaArchiveKey();

       calendar.set(Calendar.YEAR, 2008);
       calendar.set(Calendar.MONTH, Calendar.MAY);
       calendar.set(Calendar.DAY_OF_MONTH, 29);
       calendar.set(Calendar.HOUR, 10);
       calendar.set(Calendar.MINUTE, 0);
       calendar.set(Calendar.SECOND, 0);
       calendar.set(Calendar.MILLISECOND, 0);
       archiveKey.setTimePartition(calendar.getTimeInMillis());
       archiveKey.setDataType(chunk.getDataType());
       archiveKey.setStreamName(chunk.getStreamName());
       archiveKey.setSeqId(chunk.getSeqID());
       seqFileWriter.append(archiveKey, chunk);
     }
     seqFileWriter.close();
     out.close();
   }
View Full Code Here

      int ms = r.nextInt(1000);
      String line = "2008-05-29 10:42:22," + ms
          + " INFO org.apache.hadoop.dfs.DataNode: Some text goes here"
          + r.nextInt() + "\n";
      
      ChunkImpl c = new ChunkImpl("HadoopLogProcessor", "test",
      line.length()  + lastSeqID, line.getBytes(), null);
      lastSeqID += line.length();
      c.addTag("cluster=\"foocluster\"");
      return c;
    }
View Full Code Here

     */
    @Override
    public void reduce(ChukwaArchiveKey key, Iterator<ChunkImpl> vals,
        OutputCollector<ChukwaArchiveKey, ChunkImpl> out, Reporter r)
        throws IOException {
      ChunkImpl i = vals.next();
      out.collect(key, i);
      int dups = 0;
      while(vals.hasNext()) {
        vals.next();
        dups ++;
View Full Code Here

    int ms = r.nextInt(1000);
    String line = "2008-05-29 10:42:22," + ms
        + " INFO org.apache.hadoop.dfs.DataNode: Some text goes here"
        + r.nextInt() + "\n";

    ChunkImpl c = new ChunkImpl("HadoopLogProcessor", "test",
        line.length() , line.getBytes(), null);
    return c;
  }
View Full Code Here

    Calendar calendar = Calendar.getInstance();
    SequenceFile.Writer seqFileWriter = SequenceFile.createWriter(conf, out,
        ChukwaArchiveKey.class, ChunkImpl.class,
        SequenceFile.CompressionType.NONE, null);
    for (int i = 0; i < chunks; ++i) {
      ChunkImpl chunk = getARandomChunk();
      ChukwaArchiveKey archiveKey = new ChukwaArchiveKey();
      // FIXME compute this once an hour
      calendar.setTimeInMillis(System.currentTimeMillis());
      calendar.set(Calendar.MINUTE, 0);
      calendar.set(Calendar.SECOND, 0);
      calendar.set(Calendar.MILLISECOND, 0);
      archiveKey.setTimePartition(calendar.getTimeInMillis());
      archiveKey.setDataType(chunk.getDataType());
      archiveKey.setStreamName(chunk.getStreamName());
      archiveKey.setSeqId(chunk.getSeqID());
      seqFileWriter.append(archiveKey, chunk);
    }
    seqFileWriter.close();
    out.close();
  }
View Full Code Here

          continue;
       
        SequenceFile.Reader reader = new SequenceFile.Reader(localfs, fstatus.getPath(), conf);

        ChukwaArchiveKey key = new ChukwaArchiveKey();
        ChunkImpl chunk = ChunkImpl.getBlankChunk();

        while (reader.next(key, chunk)) {
         bytes.add(new ByteRange(chunk));
        }
        reader.close();
View Full Code Here

    PipelineStageWriter psw = new PipelineStageWriter();
    psw.init(conf);

    System.out.println("pipeline established; now pushing a chunk");
    ArrayList<Chunk> l = new ArrayList<Chunk>();
    l.add(new ChunkImpl("dt", "name", 1, new byte[] {'a'}, null));
    psw.add(l);
    //push a chunk through. It should get written, but the socket tee shouldn't do anything.
    assertEquals(1, CaptureWriter.outputs.size());
    //now connect and set up a filter.
   
    System.out.println("connecting to localhost");
    Socket s = new Socket("localhost", SocketTeeWriter.DEFAULT_PORT);
//   s.setSoTimeout(2000);
    DataOutputStream dos = new DataOutputStream (s.getOutputStream());
    dos.write((SocketTeeWriter.WRITABLE + " datatype=dt3\n").getBytes());
    DataInputStream dis = new DataInputStream(s.getInputStream());

    System.out.println("command send");

    dis.readFully(new byte[3]);
    //push a chunk not matching filter -- nothing should happen.
    l = new ArrayList<Chunk>();
    l.add(new ChunkImpl("dt2", "name", 1, new byte[] {'b'}, null));
    psw.add(l);
    assertEquals(2, CaptureWriter.outputs.size());

    System.out.println("sent nonmatching chunk");

    //and now one that does match -- data should be available to read off the socket

    l = new ArrayList<Chunk>();
    l.add(new ChunkImpl("dt3", "name", 1, new byte[] {'c'}, null));
    psw.add(l);
    assertEquals(3, CaptureWriter.outputs.size());

    System.out.println("sent matching chunk");
   
    System.out.println("reading...");
    ChunkImpl chunk = ChunkImpl.read(dis);
    assertTrue(chunk.getDataType().equals("dt3"));
    System.out.println(chunk);

    dis.close();
    dos.close();
    s.close();
   
    Socket s2 = new Socket("localhost", SocketTeeWriter.DEFAULT_PORT);
    s2.getOutputStream().write((SocketTeeWriter.RAW+" content=.*d.*\n").getBytes());
    dis = new DataInputStream(s2.getInputStream());
    dis.readFully(new byte[3]); //read "OK\n"
    l = new ArrayList<Chunk>();
    l.add(new ChunkImpl("dt3", "name", 1, new byte[] {'d'}, null));
    psw.add(l);
    assertEquals(4, CaptureWriter.outputs.size());

    int len = dis.readInt();
    assertTrue(len == 1);
    byte[] data = new byte[100];
    int read = dis.read(data);
    assertTrue(read == 1);
    assertTrue(data[0] == 'd');
   
    s2.close();
    dis.close();
   
    l = new ArrayList<Chunk>();
    l.add(new ChunkImpl("dt3", "name", 3, new byte[] {'c', 'a', 'd'}, null));
    psw.add(l);
    assertEquals(5, CaptureWriter.outputs.size());
   
   
    Socket s3 = new Socket("localhost", SocketTeeWriter.DEFAULT_PORT);
    s3.getOutputStream().write((SocketTeeWriter.ASCII_HEADER+" all\n").getBytes());
    dis = new DataInputStream(s3.getInputStream());
    dis.readFully(new byte[3]); //read "OK\n"
    l = new ArrayList<Chunk>();
    chunk= new ChunkImpl("dataTypeFoo", "streamName", 4, new byte[] {'t','e','x','t'}, null);
    chunk.setSource("hostNameFoo");
    l.add(chunk);
    psw.add(l);
    assertEquals(6, CaptureWriter.outputs.size());
    len = dis.readInt();
    data = new byte[len];
View Full Code Here

  public Chunk getNewChunk() {
    int ms = r.nextInt(1000);
    String line = "2008-05-29 10:42:22," + ms
        + " INFO org.apache.hadoop.dfs.DataNode: Some text goes here"
        + r.nextInt() + "\n";
    ChunkImpl c = new ChunkImpl("HadoopLogProcessor", "test",
        line.length() , line.getBytes(), null);

    return c;
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.chukwa.ChunkImpl

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.