Examples of ChukwaRecordKey

org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

          body = sb.toString();
          // Process all entries for a machine
          // System.out.println("=========>>> Record [" + body+ "]");


          record = new ChukwaRecord();
          key = new ChukwaRecordKey();


          buildGenericRecord(record, null, d.getTime(), machinePBSRecordType);
          parsePbsRecord(body, record);


          // Output PbsNode record for 1 machine
          output.collect(key, record);
          // log.info("PbsNodeProcessor output 1 sub-record");


          // compute Node Activity information
          nodeActivityStatus = record.getValue("state");
          if (nodeActivityStatus != null) {
            if (nodeActivityStatus.equals("free")) {
              totalFreeNode++;
              sbFreeMachines.append(record.getValue("Machine")).append(",");
            } else if (nodeActivityStatus.equals("job-exclusive")) {
              totalUsedNode++;
              sbUsedMachines.append(record.getValue("Machine")).append(",");
            } else {
              totalDownNode++;
              sbDownMachines.append(record.getValue("Machine")).append(",");
            }
          }
          sb = new StringBuilder();
        }
      }


      // End of parsing


      record = new ChukwaRecord();
      key = new ChukwaRecordKey();
      buildGenericRecord(record, null, d.getTime(), "NodeActivity");


      record.setTime(d.getTime());
      record.add("used", "" + totalUsedNode);
      record.add("free", "" + totalFreeNode);

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

        return;
      }
      line.setLogType("JobData");
    }
    
    key = new ChukwaRecordKey();
    ChukwaRecord record = new ChukwaRecord();
    this.buildGenericRecord(record, null, -1l, line.getLogType());
    
    for (Entry<String, String> entry : line.entrySet()) {
      record.add(entry.getKey(), entry.getValue());

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

    SequenceFile.Reader reader = null;
    log.info("File: [" +  file + "]" + fs.exists(new Path(file)));
    try {
      reader = new SequenceFile.Reader(fs, new Path(file), conf);


      ChukwaRecordKey key = new ChukwaRecordKey();
      ChukwaRecord record = new ChukwaRecord();


      StringBuilder sb = new StringBuilder();
      while (reader.next(key, record)) {
       
        sb.append("===== KEY   =====");


        sb.append("DataType: " + key.getReduceType());
        sb.append("Key: " + key.getKey());
        sb.append("===== Value =====");


        String[] fields = record.getFields();
        Arrays.sort(fields );
        sb.append("Timestamp : " + record.getTime());

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

    SequenceFile.Reader reader = null;
    log.info("File: [" +  file + "]" + fs.exists(new Path(file)));
    try {
      reader = new SequenceFile.Reader(fs, new Path(file), conf);


      ChukwaRecordKey key = new ChukwaRecordKey();
      ChukwaRecord record = new ChukwaRecord();


      StringBuilder sb = new StringBuilder();
      while (reader.next(key, record)) {
       
        sb.append("===== KEY   =====");


        sb.append("DataType: " + key.getReduceType());
        sb.append("Key: " + key.getKey());
        sb.append("===== Value =====");


        String[] fields = record.getFields();
        Arrays.sort(fields );
        sb.append("Timestamp : " + record.getTime());

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

      System.exit(-1);
    }
    System.out.println("outputFile:" + outputFile);
    
    SequenceFile.Writer seqFileWriter = SequenceFile.createWriter(fs, conf,outputFile, ChukwaRecordKey.class,ChukwaRecord.class,CompressionType.NONE);
    ChukwaRecordKey key = new ChukwaRecordKey();
    key.setReduceType("TestSeqFile");
    
   
    
    String chukwaKey = "";
    String machine = "";
    String TimePartion = "1242205200"; //Wed, 13 May 2009 09:00:00 GMT
    
    {    
      
      machine = "M0";
      long time = 1242205800; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "7");
      record.add("B", "3");
      record.add("C", "9");
  
      seqFileWriter.append(key, record);
    }


    {    
      machine = "M0";
      long time = 1242205800; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("D", "1");
  
      seqFileWriter.append(key, record);
    }


    {    
      machine = "M1";
      long time = 1242205800; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "17");
  
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M1";
      long time = 1242205800; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("B", "37");
      record.add("C", "51");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M0";
      long time = 1242205860; // Wed, 13 May 2009 09:10:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8");
      record.add("C", "3");
      record.add("D", "12");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M0";
      long time = 1242205860; // Wed, 13 May 2009 09:11:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8");
      record.add("B", "6");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M1";
      long time = 1242205860; // Wed, 13 May 2009 09:11:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "13.2");
      record.add("B", "23");
      record.add("C", "8.5");
      record.add("D", "6");
      
      // create duplicate
      seqFileWriter.append(key, record);
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M0";
      long time = 1242205920; // Wed, 13 May 2009 09:12:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8");
      record.add("B", "6");
      record.add("C", "8");
      record.add("D", "6");
      record.add("E", "48.5");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M1";
      long time = 1242205920; // Wed, 13 May 2009 09:12:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8.3");
      record.add("B", "5.2");
      record.add("C", "37.7");
      record.add("D", "61.9");
      record.add("E", "40.3");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M1";
      long time = 1242205980; // Wed, 13 May 2009 09:13:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "18.3");
      record.add("B", "1.2");
      record.add("C", "7.7");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M2";
      long time = 1242205980; // Wed, 13 May 2009 09:13:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "8.9");
      record.add("B", "8.3");
      record.add("C", "7.2");
      record.add("D", "6.1");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M3";
      // late arrival T0
      long time = 1242205920; // Wed, 13 May 2009 09:12:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "12.5");
      record.add("B", "26.82");
      record.add("C", "89.51");
      seqFileWriter.append(key, record);
    }
    
    {    
      machine = "M4";
      // late arrival T0
      long time = 1242205920; // Wed, 13 May 2009 09:12:00 GMT
      chukwaKey = TimePartion +"/" + machine +"/" + time;
      key.setKey(chukwaKey);
      
      ChukwaRecord record = new ChukwaRecord();
      record.setTime(time);
      record.add("csource", machine);
      record.add("A", "13.91");

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

    String pk = "";
    
    try
    {


      ChukwaRecordKey key = new ChukwaRecordKey();
     


      ChukwaRecord record = new ChukwaRecord();
      record.setTime(System.currentTimeMillis());
      int inputSize = f.size();      
      for(int i=0;i<inputSize;i++) {
        Object field = f.get(i);
        
        if (field == null) {
          continue;
        }
        
        if (i == this.pkFieldIndex) {
          pk = field.toString();
          continue;
        } else if ( i == this.sourceFieldIndex) {
          source = field.toString();
          continue;
          
        }else if ( i== this.recordTypeFieldIndex) {
          recordType = field.toString();
          continue;
          
        }else if ( i== this.applicationFieldIndex) {
          application = field.toString();
          continue;
          
        } else if ( i== this.clusterNameFieldIndex) {
          clusterName = field.toString();
          continue;
          
        }else if (i == this.timestampFieldIndex) {
          
          timestamp = Long.parseLong(field.toString());
          record.setTime(timestamp);


          synchronized (calendar)
          {
            calendar.setTimeInMillis(timestamp);
            calendar.set(Calendar.MINUTE, 0);
            calendar.set(Calendar.SECOND, 0);
            calendar.set(Calendar.MILLISECOND, 0);
            timePartition = calendar.getTimeInMillis();
          }
          record.setTime(Long.parseLong(field.toString()));
          continue;
          
        }  else if (field instanceof Map) {
          Map<Object, Object> m = (Map<Object, Object>)field;
          for(Object o: m.keySet()) {
            record.add(o.toString(),m.get(o).toString());
          }
          continue;
        } else {
          if (i <fields.length ) {
            record.add(fields[i],field.toString());
          } else {
            record.add("field-"+i,field.toString());
          }
          
          continue;
        }
      }


      record.add(Record.tagsField, " cluster=\"" + clusterName.trim() + "\" ");
      record.add(Record.sourceField, source);
      record.add(Record.applicationField, application);
      key.setKey("" + timePartition + "/" + pk + "/" + timestamp);
      key.setReduceType(recordType);
      
      writer.append(key, record);
    } catch (ExecException e)
    {
      IOException ioe = new IOException();

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

      }
      log.debug("Parser Open [" + fileName + "]");


      long timestamp = 0;
      int listSize = 0;
      ChukwaRecordKey key = new ChukwaRecordKey();
      ChukwaRecord record = new ChukwaRecord();


      r = new SequenceFile.Reader(fs, new Path(fileName), conf);


      log.debug("readData Open2 [" + fileName + "]");
      if ((fileName.equals(res.fileName)) && (res.position != -1)) {
        r.seek(res.position);
      }
      res.fileName = fileName;


      while (r.next(key, record)) {
        if (record != null) {
          res.position = r.getPosition();


          timestamp = record.getTime();
          res.currentTs = timestamp;
          log.debug("\nSearch for startDate: " + new Date(t0) + " is :"
              + new Date(timestamp));


          if (timestamp < t0) {
            // log.debug("Line not in range. Skipping: " +record);
            continue;
          } else if (timestamp < t1) {
            log.debug("In Range: " + record.toString());
            boolean valid = false;


            if ((filter == null || filter.equals(""))) {
              valid = true;
            } else if (isValid(record, filter)) {
              valid = true;
            }


            if (valid) {
              records.add(record);
              record = new ChukwaRecord();
              listSize = records.size();
              if (listSize >= maxRows) {
                // maxRow so stop here
                // Update token
                token.key = key.getKey();
                token.hasMore = true;
                break;
              }
            } else {
              log.debug("In Range ==================>>>>>>>>> OUT Regex: "
                  + record);
            }
          } else {
            log.debug("Line out of range. Stopping now: " + record);
            // Update Token
            token.key = key.getKey();
            token.hasMore = false;
            break;
          }
        }
      }

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

          // FIXME please validate this
          if (headers[1].equals("IFACE") && headers[2].equals("rxpck/s")) {
            log.debug("Matched Sar-Network");


            record = new ChukwaRecord();
            key = new ChukwaRecordKey();
            this.buildGenericRecord(record, null, d.getTime(), "SystemMetrics");
          } else if (headers[1].equals("IFACE") && headers[2].equals("rxerr/s")) {
            log.debug("Matched Sar-Network");


            record = new ChukwaRecord();
            key = new ChukwaRecordKey();
            this.buildGenericRecord(record, null, d.getTime(), "SystemMetrics");
          } else if (headers[1].equals("kbmemfree")) {
            log.debug("Matched Sar-Memory");


            record = new ChukwaRecord();
            key = new ChukwaRecordKey();
            this.buildGenericRecord(record, null, d.getTime(), "SystemMetrics");
          } else if (headers[1].equals("totsck")) {
            log.debug("Matched Sar-NetworkSockets");


            record = new ChukwaRecord();
            key = new ChukwaRecordKey();
            this.buildGenericRecord(record, null, d.getTime(), "SystemMetrics");
          } else if (headers[1].equals("runq-sz")) {
            log.debug("Matched Sar-LoadAverage");


            record = new ChukwaRecord();
            key = new ChukwaRecordKey();
            this.buildGenericRecord(record, null, d.getTime(), "SystemMetrics");
          } else {
            log.debug("No match:" + headers[1] + " " + headers[2]);
          }
          if (record != null) {

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

    conn.setAutoCommit(false);
    long currentTimeMillis = System.currentTimeMillis();
    boolean isSuccessful = true;
    String recordType = null;


    ChukwaRecordKey key = new ChukwaRecordKey();
    ChukwaRecord record = new ChukwaRecord();
    try {
      int batch = 0;
      while (reader.next(key, record)) {
        String sqlTime = DatabaseWriter.formatTimeStamp(record.getTime());
        log.debug("Timestamp: " + record.getTime());
        log.debug("DataType: " + key.getReduceType());


        String[] fields = record.getFields();
        String table = null;
        String[] priKeys = null;
        HashMap<String, HashMap<String, String>> hashReport = new HashMap<String, HashMap<String, String>>();
        StringBuilder normKey = new StringBuilder();
        String node = record.getValue("csource");
        recordType = key.getReduceType().toLowerCase();
        String dbKey = "report.db.name." + recordType;
        if (dbTables.containsKey(dbKey)) {
          String[] tmp = mdlConfig.findTableName(mdlConfig.get(dbKey), record
              .getTime(), record.getTime());
          table = tmp[0];

View Full Code Here

Examples of org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey

  // [MAP-REDUCE_FRAMEWORK_REDUCE_INPUT_RECORDS] :739000
  // [MAP-REDUCE_FRAMEWORK_REDUCE_OUTPUT_RECORDS] :739000


  @Override
  public void process(Reader reader) throws DBException {
    ChukwaRecordKey key = new ChukwaRecordKey();
    ChukwaRecord record = new ChukwaRecord();
    try {
      StringBuilder sb = new StringBuilder();
      while (reader.next(key, record)) {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.