Package org.apache.hadoop.chukwa.conf

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  /**
   * No dataSink file at  startup
   * Add one later
   */
  public void testScenario2() {
    ChukwaConfiguration cc = new ChukwaConfiguration();
    String tempDirectory = System.getProperty("test.build.data", "/tmp");
    String chukwaRootDir = tempDirectory + "/demuxManagerTest_" + System.currentTimeMillis() +"/";
   
    cc.set(CHUKWA_CONSTANT.HDFS_DEFAULT_NAME_FIELD, "file:///");
    cc.set(CHUKWA_CONSTANT.CHUKWA_ROOT_DIR_FIELD, chukwaRootDir );
    cc.set(CHUKWA_CONSTANT.CHUKWA_ARCHIVE_DIR_FIELD, chukwaRootDir +"/archives/" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_POST_PROCESS_DIR_FIELD, chukwaRootDir +"/postProcess" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_DATA_SINK_DIR_FIELD, chukwaRootDir +"/logs" );

    try {
      DemuxManagerScenario dm = new DemuxManagerScenario2(cc); 
      dm.start();
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
   * MR_INPUT_DIR already there
   * MR_INPUT_DIR should be reprocessed first
   * Then dataSink file
   */
  public void testScenario3() {
    ChukwaConfiguration cc = new ChukwaConfiguration();
    String tempDirectory = System.getProperty("test.build.data", "/tmp");
    String chukwaRootDir = tempDirectory + "/demuxManagerTest_" + System.currentTimeMillis() +"/";
   
    cc.set(CHUKWA_CONSTANT.HDFS_DEFAULT_NAME_FIELD, "file:///");
    cc.set(CHUKWA_CONSTANT.CHUKWA_ROOT_DIR_FIELD, chukwaRootDir );
    cc.set(CHUKWA_CONSTANT.CHUKWA_ARCHIVE_DIR_FIELD, chukwaRootDir +"/archives/" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_POST_PROCESS_DIR_FIELD, chukwaRootDir +"/postProcess" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_DATA_SINK_DIR_FIELD, chukwaRootDir +"/logs" );

    try {
      File mrInputDir = new File(chukwaRootDir + CHUKWA_CONSTANT.DEFAULT_DEMUX_PROCESSING_DIR_NAME+ CHUKWA_CONSTANT.DEFAULT_DEMUX_MR_INPUT_DIR_NAME);
      mrInputDir.mkdirs();
      File dataSinkDirectory = new File(chukwaRootDir +"/logs");
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
   * MR_OUTPUT_DIR should be deleted first
   * Then MR_INPUT_DIR should be reprocessed
   * Then dataSink file
   */
  public void testScenario4() {
    ChukwaConfiguration cc = new ChukwaConfiguration();
    String tempDirectory = System.getProperty("test.build.data", "/tmp");
    String chukwaRootDir = tempDirectory + "/demuxManagerTest_" + System.currentTimeMillis() +"/";
   
    cc.set(CHUKWA_CONSTANT.HDFS_DEFAULT_NAME_FIELD, "file:///");
    cc.set(CHUKWA_CONSTANT.CHUKWA_ROOT_DIR_FIELD, chukwaRootDir );
    cc.set(CHUKWA_CONSTANT.CHUKWA_ARCHIVE_DIR_FIELD, chukwaRootDir +"/archives/" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_POST_PROCESS_DIR_FIELD, chukwaRootDir +"/postProcess" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_DATA_SINK_DIR_FIELD, chukwaRootDir +"/logs" );

    try {
      File mrInputDir = new File(chukwaRootDir + CHUKWA_CONSTANT.DEFAULT_DEMUX_PROCESSING_DIR_NAME+ CHUKWA_CONSTANT.DEFAULT_DEMUX_MR_INPUT_DIR_NAME);
      mrInputDir.mkdirs();
      File mrOutputDir = new File(chukwaRootDir + CHUKWA_CONSTANT.DEFAULT_DEMUX_PROCESSING_DIR_NAME+ CHUKWA_CONSTANT.DEFAULT_DEMUX_MR_OUTPUT_DIR_NAME);
View Full Code Here
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
   * Demux fails 3 times
   * Add a new DataSink file to dataSinkDir
   * Demux succeed
   */
  public void testScenario5() {
    ChukwaConfiguration cc = new ChukwaConfiguration();
    String tempDirectory = System.getProperty("test.build.data", "/tmp");
    String chukwaRootDir = tempDirectory + "/demuxManagerTest_" + System.currentTimeMillis() +"/";
   
    cc.set(CHUKWA_CONSTANT.HDFS_DEFAULT_NAME_FIELD, "file:///");
    cc.set(CHUKWA_CONSTANT.CHUKWA_ROOT_DIR_FIELD, chukwaRootDir );
    cc.set(CHUKWA_CONSTANT.CHUKWA_ARCHIVE_DIR_FIELD, chukwaRootDir +"/archives/" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_POST_PROCESS_DIR_FIELD, chukwaRootDir +"/postProcess" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_DATA_SINK_DIR_FIELD, chukwaRootDir +"/logs" );

    try {
      File dataSinkDirectory = new File(chukwaRootDir +"/logs");
      dataSinkDirectory.mkdirs();
      File dataSinkFile = new File(chukwaRootDir +"/logs"+ "/dataSink5-0.done");
View Full Code Here
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
 
  /**
   * Standard workflow with MR_OUTPUT
   */
  public void testScenario6() {
    ChukwaConfiguration cc = new ChukwaConfiguration();
    String tempDirectory = System.getProperty("test.build.data", "/tmp");
    String chukwaRootDir = tempDirectory + "/demuxManagerTest_" + System.currentTimeMillis() +"/";
   
   
    cc.set(CHUKWA_CONSTANT.WRITER_HDFS_FILESYSTEM_FIELD, "file:///");
    cc.set(CHUKWA_CONSTANT.CHUKWA_ROOT_DIR_FIELD, chukwaRootDir );
    cc.set(CHUKWA_CONSTANT.CHUKWA_ARCHIVE_DIR_FIELD, chukwaRootDir +"/archives/" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_POST_PROCESS_DIR_FIELD, chukwaRootDir +"/postProcess" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_DATA_SINK_DIR_FIELD, chukwaRootDir +"/logs" );
   
    try {
     
      File dataSinkDirectory = new File(chukwaRootDir +"/logs");
      dataSinkDirectory.mkdirs();
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
      ChukwaAgent agent = new ChukwaAgent();
      HttpConnector connector = new HttpConnector(agent,
          "http://localhost:9990/chukwa");
      connector.start();

      ChukwaConfiguration cc = new ChukwaConfiguration();
      int portno = cc.getInt("chukwaAgent.control.port", 9093);
      ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);

      File workdir = new File("/tmp/stresstest/");
      workdir.mkdir();
      for (int i = 0; i < FILES_TO_USE; ++i) {
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
  public void testLogRotate() throws IOException, InterruptedException,
      ChukwaAgent.AlreadyRunningException {
    ChukwaAgent agent = new ChukwaAgent();
    // Remove any adaptor left over from previous run
    ChukwaConfiguration cc = new ChukwaConfiguration();
    int portno = cc.getInt("chukwaAgent.control.port", 9093);
    ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);
    cli.removeAll();
    // sleep for some time to make sure we don't get chunk from existing streams
    Thread.sleep(5000);
    File testFile = makeTestFile("chukwaLogRotateTest", 80);
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
  /**
   * @param args
   * @throws Exception
   */
  public static void main(String[] args) throws Exception {
    conf = new ChukwaConfiguration();
    String fsName = conf.get("writer.hdfs.filesystem");
    fs = FileSystem.get(new URI(fsName), conf);

    Path srcDir = new Path(args[0]);
    String destDir = args[1];
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
    ToolRunner.printGenericCommandUsage(System.out);
    return -1;
  }

  public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(new ChukwaConfiguration(), Demux.class);
   

    conf.setJobName("Chukwa-Demux_" + day.format(new Date()));
    conf.setInputFormat(SequenceFileInputFormat.class);
    conf.setMapperClass(Demux.MapClass.class);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    }
    if(args[0].equals("--summarize")) {
      firstArg = 1;
      summarize= true;
    }
    ChukwaConfiguration conf = new ChukwaConfiguration();
    FileSystem fs;
    if(args[firstArg].contains("://")) {
      fs = FileSystem.get(new URI(args[firstArg]), conf);
    } else {
      String fsName = conf.get("writer.hdfs.filesystem");
      if(fsName != null)
        fs = FileSystem.get(conf);
      else
        fs = FileSystem.getLocal(conf);
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

  • hitune.analysis.mapreduce.Analysis
  • org.apache.hadoop.chukwa.analysis.salsa.fsm.TestFSMBuilder
  • org.apache.hadoop.chukwa.database.TestDatabaseAggregator
  • org.apache.hadoop.chukwa.database.TestDatabaseDataExpiration
  • org.apache.hadoop.chukwa.database.TestDatabaseIostat
  • org.apache.hadoop.chukwa.database.TestDatabaseTableCreator
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileExpirationPolicy
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailer
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailingAdaptorBigRecord
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailingAdaptorPreserveLines
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.