Package org.apache.hadoop.chukwa.datacollection.agent

Examples of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent


  int SAMPLE_PERIOD_MS = DEFAULT_SAMPLE_PERIOD_MS;
  private static Configuration conf = null;

  FileTailer() {
    if (conf == null) {
      ChukwaAgent agent = ChukwaAgent.getAgent();
      if (agent != null) {
        conf = agent.getConfiguration();
        if (conf != null) {
          SAMPLE_PERIOD_MS = conf.getInt(
              "chukwaAgent.adaptor.context.switch.time",
              DEFAULT_SAMPLE_PERIOD_MS);
        }
View Full Code Here


      reader = null;
    }
  }
 
  private void unregisterFromAgent() {
    ChukwaAgent agent = ChukwaAgent.getAgent();
    if (agent != null) {
      agent.stopAdaptor(adaptorID, false);
     
    } else {
      log.warn("Agent is null, cannot unregister " + toWatch.getAbsolutePath());
    }
View Full Code Here

          gracefulPeriodExpired = 0L;
          attempts = 0;
          return false;
        }

        ChukwaAgent agent = ChukwaAgent.getAgent();
        if (agent != null) {
          agent.stopAdaptor(adaptorID, false);
        } else {
          log.info("Agent is null, running in default mode");
        }
        return false;

      } else if (!toWatch.exists() || !toWatch.canRead()) {
        if (adaptorInError == false) {
          long now = System.currentTimeMillis();
          gracefulPeriodExpired = now + GRACEFUL_PERIOD;
          adaptorInError = true;
          attempts = 0;
          log.warn("failed to stream data for: " + toWatch.getAbsolutePath()
              + ", graceful period will Expire at now:" + now + " + "
              + GRACEFUL_PERIOD + " secs, i.e:" + gracefulPeriodExpired);
        } else if (attempts % 10 == 0) {
          log.info("failed to stream data for: " + toWatch.getAbsolutePath()
              + ", attempt: " + attempts);
        }

        attempts++;
        return false; // no more data
      }

      if (reader == null) {
        reader = new RandomAccessFile(toWatch, "r");
        log.info("Adaptor|" + adaptorID
            + "|Opening the file for the first time|seek|" + fileReadOffset);
      }

      long len = 0L;
      try {
        RandomAccessFile newReader = new RandomAccessFile(toWatch, "r");
        len = reader.length();
        long newLength = newReader.length();
        if (newLength < len && fileReadOffset >= len) {
          if (reader != null) {
            reader.close();
          }
         
          reader = newReader;
          fileReadOffset = 0L;
          log.debug("Adaptor|" + adaptorID
              + "| File size mismatched, rotating: "
              + toWatch.getAbsolutePath());
        } else {
          try {
            if (newReader != null) {
              newReader.close();
            }
            newReader =null;
          } catch (Throwable e) {
            // do nothing.
          }
        }
      } catch (IOException e) {
        // do nothing, if file doesn't exist.
      }
      if (len >= fileReadOffset) {
        if (offsetOfFirstByte > fileReadOffset) {
          // If the file rotated, the recorded offsetOfFirstByte is greater than
          // file size,
          // reset the first byte position to beginning of the file.
          fileReadOffset = 0;
          offsetOfFirstByte = 0L;
          log.warn("offsetOfFirstByte>fileReadOffset, resetting offset to 0");
        }

        log.debug("Adaptor|" + adaptorID + "|seeking|" + fileReadOffset);
        reader.seek(fileReadOffset);

        long bufSize = len - fileReadOffset;

        if (conf == null) {
          ChukwaAgent agent = ChukwaAgent.getAgent();
          if (agent != null) {
            conf = agent.getConfiguration();
            if (conf != null) {
              MAX_READ_SIZE = conf.getInt(
                  "chukwaAgent.fileTailingAdaptor.maxReadSize",
                  DEFAULT_MAX_READ_SIZE);
              log.info("chukwaAgent.fileTailingAdaptor.maxReadSize: "
View Full Code Here

          new ChukwaConfiguration(true))), "/*");
      server.start();
      server.setStopAtShutdown(false);

      Thread.sleep(1000);
      ChukwaAgent agent = new ChukwaAgent();
      HttpConnector connector = new HttpConnector(agent,
          "http://localhost:9990/chukwa");
      connector.start();

      ChukwaConfiguration cc = new ChukwaConfiguration();
View Full Code Here

public class TestAgent extends TestCase {

  public void testStopAndStart() {

    try {
      ChukwaAgent agent = new ChukwaAgent();
      ConsoleOutConnector conn = new ConsoleOutConnector(agent, true);
      conn.start();

      ChukwaConfiguration cc = new ChukwaConfiguration();
      int portno = cc.getInt("chukwaAgent.control.port", 9093);
      ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);

      for (int i = 1; i < 20; ++i) {
        long adaptorId = cli.add(
            "org.apache.hadoop.chukwa.util.ConstRateAdaptor", "raw" + i, "2000"
                + i, 0);
        assertTrue(adaptorId != -1);
        Thread.sleep(2000);
        cli.removeAll();
      }
      agent.shutdown();
      conn.shutdown();
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.toString());
    }
View Full Code Here

  }

  public void testMultiStopAndStart() {

    try {
      ChukwaAgent agent = new ChukwaAgent();
      ConsoleOutConnector conn = new ConsoleOutConnector(agent, true);
      conn.start();
      int count = agent.adaptorCount();
      for (int trial = 0; trial < 20; ++trial) {
        ArrayList<Long> runningAdaptors = new ArrayList<Long>();

        for (int i = 1; i < 7; ++i) {
          long l = agent
              .processCommand("add  org.apache.hadoop.chukwa.util.ConstRateAdaptor  raw"
                  + i + " 2000" + i + " 0");
          assertTrue(l != -1);
          runningAdaptors.add(l);
        }
        Thread.sleep(1000);
        for (Long l : runningAdaptors)
          agent.stopAdaptor(l, false);
        Thread.sleep(5000);
        assertTrue(agent.adaptorCount() == count);
      }
      agent.shutdown();
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.toString());
    }
  }
View Full Code Here

    // Start the collector
    System.out.println("Starting collector");
    CollectorStub.main(new String[0]);

    // Start the agent
    ChukwaAgent agent = ChukwaAgent.getAgent();

    int portno = 9093; // Default
    ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);
    // ADD
    // org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.
    // CharFileTailingAdaptorUTF8NewLineEscaped
    // SysLog
    // 0 /var/log/messages
    // 0
    System.out.println("Adding adaptor");
    long adaptor = cli
        .add(
            "org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped",
            "AutomatedTestType", "0 " + inputFile, 0);

    cli.remove(adaptor);
    System.out.println("Adaptor removed");
    agent.shutdown();
    System.out.println("Shutting down agent");
    CollectorStub.jettyServer.stop();
    System.out.println("Shutting down collector");
    Thread.sleep(2000);
  }
View Full Code Here

  // consoleConnector = new ConsoleOutConnector(agent);

  protected void setUp() throws ChukwaAgent.AlreadyRunningException {
    config = new ChukwaConfiguration();
    agent = new ChukwaAgent();
    c = new ChukwaAgentController();
    httpConnector = new HttpConnector(agent); // use default source for list of
                                              // collectors (i.e.
                                              // conf/connectors)
View Full Code Here

    chunks.start();
  }

  public void testWithPs() throws ChukwaAgent.AlreadyRunningException {
    try {
      ChukwaAgent agent = new ChukwaAgent();
      agent
          .processCommand("add org.apache.hadoop.chukwa.datacollection.adaptor.ExecAdaptor ps ps aux 0");

      Chunk c = chunks.waitForAChunk();
      System.out.println(new String(c.getData()));
    } catch (InterruptedException e) {
View Full Code Here

public class TestFailedCollector extends TestCase {

  public void testFailedCollector() {
    try {
      ChukwaAgent agent = new ChukwaAgent();
      boolean failed = false;
      HttpConnector connector = new HttpConnector(agent,
          "http://localhost:1234/chukwa");
      connector.start();

      ChukwaConfiguration cc = new ChukwaConfiguration();
      int portno = cc.getInt("chukwaAgent.control.port", 9093);
      ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);

      File tmpOutput = TempFileUtil.makeBinary(2000);

      cli.addFile("unknown", tmpOutput.getAbsolutePath());
      System.out.println("have " + agent.adaptorCount() + " running adaptors");
      cli.removeFile("unknown", tmpOutput.getAbsolutePath());

      tmpOutput.delete();
      assertFalse(failed);
      System.out.println("done");
      agent.shutdown();
      connector.shutdown();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.