Examples of ChukwaConfiguration

org.apache.hadoop.chukwa.conf.ChukwaConfiguration

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

  
  static Logger log = Logger.getLogger(SinkArchiver.class);
  
  public static void main(String[] args) {
    try {
      Configuration conf = new ChukwaConfiguration();
      if(conf.get(ChukwaArchiveDataTypeOutputFormat.GROUP_BY_CLUSTER_OPTION_NAME) == null )
        conf.set(ChukwaArchiveDataTypeOutputFormat.GROUP_BY_CLUSTER_OPTION_NAME, "true");
      FileSystem fs = FileSystem.get(conf);
      SinkArchiver archiver = new SinkArchiver();
      archiver.exec(fs, conf);    
    } catch(IOException e) {
      e.printStackTrace();

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

    JobClient.runJob(jobConf);
    return 0;
  }


  public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new ChukwaConfiguration(), new ChukwaArchiveBuilder(),
        args);
    System.exit(res);
  }

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

  protected void setUp() throws Exception {
    MockTriggerAction.reset();
  }


  public void testSuccessTrigger() throws Exception {
    ChukwaConfiguration conf = new ChukwaConfiguration();
    conf.set(CHUKWA_CONSTANT.POST_DEMUX_SUCCESS_ACTION,
            "org.apache.hadoop.chukwa.extraction.demux.MockTriggerAction");


    PostProcessorManager postProcessManager = new PostProcessorManager(conf);
    assertTrue("processPostMoveTriggers returned false",
            postProcessManager.processPostMoveTriggers(SAMPLE_PATHS));

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

    assertEquals("Incorrect Trigger event found", TriggerEvent.POST_DEMUX_SUCCESS,
            events.next());
  }


  public void testMultiSuccessTrigger() throws Exception {
    ChukwaConfiguration conf = new ChukwaConfiguration();
    conf.set(CHUKWA_CONSTANT.POST_DEMUX_SUCCESS_ACTION,
            "org.apache.hadoop.chukwa.extraction.demux.MockTriggerAction," +
            "org.apache.hadoop.chukwa.extraction.demux.MockTriggerAction");


    PostProcessorManager postProcessManager = new PostProcessorManager(conf);
    assertTrue("processPostMoveTriggers returned false",

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

      System.setProperty("hadoop.log.dir", System.getProperty(
          "test.build.data", "/tmp"));
    
      String[] sortArgs = { DEMUX_INPUT_PATH.toString(), DEMUX_OUTPUT_PATH.toString() };
      //      JobConf job = mr.createJobConf();
      JobConf job = new JobConf(new ChukwaConfiguration(), Demux.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-Demux_" + day.format(new Date()));
      job.setInputFormat(SequenceFileInputFormat.class);
      job.setMapperClass(Demux.MapClass.class);
      job.setPartitionerClass(ChukwaRecordPartitioner.class);

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

    // Test FSMBuilder (job history only)
    log.info("Testing FSMBuilder (Job History only)");
    System.out.println("In JobHistory020");
    // Run FSMBuilder on Demux output
    try {
      JobConf job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(JobHistoryTaskDataMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

    log.info("Testing FSMBuilder (ClientTrace only)");
    System.out.println("In ClientTrace020");
    // Run FSMBuilder on Demux output
    try {
      // Process TaskTracker shuffle clienttrace entries first
      JobConf job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(TaskTrackerClientTraceMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);
      job.setMapOutputValueClass(FSMIntermedEntry.class);
      job.setMapOutputKeyClass(ChukwaRecordKey.class);


      job.setInputFormat(SequenceFileInputFormat.class);
      job.setOutputKeyClass(ChukwaRecordKey.class);
      job.setOutputValueClass(ChukwaRecord.class);
      job.setOutputFormat(ChukwaRecordOutputFormat.class);
      job.setNumReduceTasks(1);


      Path inputPath = new Path(DEMUX_OUTPUT_PATH.toString()+File.separator+"/*/*/ClientTraceDetailed*.evt");
      Path fsmOutputPath1 = new Path(fileSys.getUri().toString()+File.separator+fsmSink+"1");


      FileInputFormat.setInputPaths(job, inputPath);
      FileOutputFormat.setOutputPath(job, fsmOutputPath1);


      String[] jars = new File(System.getenv("CHUKWA_HOME")).list(new FilenameFilter() {
        public boolean accept(File dir, String name) {
          return name.endsWith(".jar");
        }
      });
      job.setJar(System.getenv("CHUKWA_HOME")+File.separator+jars[0]);
      JobClient.runJob(job);
      System.out.println("Processed TaskTracker ClientTrace.");


      // Process DataNode clienttrace entries
      job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(DataNodeClientTraceMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

      boolean failed = false;
      HttpConnector connector = new HttpConnector(agent,
          "http://localhost:1234/chukwa");
      connector.start();


      ChukwaConfiguration cc = new ChukwaConfiguration();
      int portno = cc.getInt("chukwaAgent.control.port", 9093);
      ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);


      File tmpOutput = TempFileUtil.makeBinary(2000);


      cli.addFile("unknown", tmpOutput.getAbsolutePath());

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

  private byte[] test = Bytes.toBytes("1234567890 Key Value");
  private ChukwaConfiguration cc;
  long timestamp = 1234567890;
  
  public TestHBaseWriter() {
    cc = new ChukwaConfiguration();


    conf = HBaseConfiguration.create();
    conf.set("hbase.hregion.memstore.flush.size", String.valueOf(128*1024));
    try {
      util = new HBaseTestingUtility(conf);

View Full Code Here

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration


  /**
   * Standard workflow
   */
  public void testScenario1() {
    ChukwaConfiguration cc = new ChukwaConfiguration();
    String tempDirectory = System.getProperty("test.build.data", "/tmp");
    String chukwaRootDir = tempDirectory + "/demuxManagerTest_" + System.currentTimeMillis() +"/";
    
    cc.set(CHUKWA_CONSTANT.HDFS_DEFAULT_NAME_FIELD, "file:///");
    cc.set(CHUKWA_CONSTANT.CHUKWA_ROOT_DIR_FIELD, chukwaRootDir );
    cc.set(CHUKWA_CONSTANT.CHUKWA_ARCHIVE_DIR_FIELD, chukwaRootDir +"/archives/" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_POST_PROCESS_DIR_FIELD, chukwaRootDir +"/postProcess" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_DATA_SINK_DIR_FIELD, chukwaRootDir +"/logs" );
    
     
    try {
      
      File dataSinkDirectory = new File(chukwaRootDir +"/logs");

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.