Package com.cloudera.flume.core

Examples of com.cloudera.flume.core.EventSink

295
296
297
298
299
300
301
302
303
304
305
   */
  synchronized public EventSink newWritingSink(final Tagger tagger)
      throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();
    EventSink curSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    writingQ.add(tag);
    WALData data = new WALData(tag);
    table.put(tag, data);

View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
      InterruptedException {
    BenchmarkHarness.setupLocalWriteDir();
    // String spec =
    // "{ benchinject => { benchreport(\"pre\") =>  { diskFailover => [console, counter(\"beforecount\")] } } }";
    String spec = "{ benchinject => { benchreport(\"pre\") =>  { diskFailover => counter(\"beforecount\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), spec);

    EventSource src = MemorySinkSource.cannedData("test ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
        "beforecount");
    Assert.assertEquals(5, cnt.getCount());
    BenchmarkHarness.cleanupLocalWriteDir();
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
  @Test
  public void benchmarkAfterFailover() throws FlumeSpecException, IOException,
      InterruptedException {
    BenchmarkHarness.setupLocalWriteDir();
    String spec = "{ benchinject => { diskFailover => { benchreport(\"post\") =>  counter(\"beforecount\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), spec);

    EventSource src = MemorySinkSource.cannedData("test ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
        "beforecount");
    Assert.assertEquals(5, cnt.getCount());
    BenchmarkHarness.cleanupLocalWriteDir();
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
  @Test
  public void benchmarkBeforeWriteahead() throws FlumeSpecException,
      IOException, InterruptedException {
    BenchmarkHarness.setupLocalWriteDir();
    String spec = "{ benchinject => { benchreport(\"pre\") =>  { diskFailover => counter(\"beforecount\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), spec);

    EventSource src = MemorySinkSource.cannedData("test ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
        "beforecount");
    Assert.assertEquals(5, cnt.getCount());
    BenchmarkHarness.cleanupLocalWriteDir();
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
  public void benchmarkAfterWriteahead() throws FlumeSpecException,
      IOException, InterruptedException {
    BenchmarkHarness.setupLocalWriteDir();

    String spec = "{ benchinject => { ackedWriteAhead => { benchreport(\"post\") =>  counter(\"beforecount\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), spec);

    EventSource src = MemorySinkSource.cannedData("test ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
        "beforecount");

    // +2 because of wal ack begin and end messages.
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
  @SuppressWarnings("unchecked")
  @Test
  public void testBloomDecos() throws FlumeSpecException, IOException,
      InterruptedException {
    String spec = "{ bloomGen(10000,2) => { bloomCheck(10000,2) => counter(\"test\")} } ";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
    EventSource src = FlumeBuilder.buildSource(LogicalNodeContext
        .testingContext(), "asciisynth(10000)");
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
    assertEquals(ctr.getCount(), 10000);

    // Hack until we get a better mechanism:
View Full Code Here
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
   */
  @Test
  public void testBloomReportSink() throws FlumeSpecException, IOException,
      InterruptedException {
    String spec = "{bloomGen(100,2) => {bloomCheck(100,2,\"counter(\\\"test\\\") \")  => counter(\"total\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
    snk.open();
    snk.append(new EventImpl(new byte[0]));
    snk.append(new EventImpl(new byte[0]));

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
    assertEquals(0, ctr.getCount());
    CounterSink total = (CounterSink) ReportManager.get()
        .getReportable("total");
    assertEquals(2, total.getCount());

    snk.close(); // will trigger a bloom report.
    assertEquals(1, ctr.getCount());
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
   */
  static EventSink buildSpec(Context context, String spec, List<String> list,
      BackoffPolicy policy) throws FlumeSpecException {

    // iterate through the list backwards
    EventSink cur = null;
    for (int i = list.size() - 1; i >= 0; i--) {
      String s = list.get(i);
      // this should be a composite sink.
      String failoverSpec = String.format(spec, s);
      LOG.debug("failover spec is : " + failoverSpec);
      EventSink tsnk = new CompositeSink(context, failoverSpec);
      if (cur == null) {
        cur = tsnk;
        continue;
      }
      cur = new BackOffFailOverSink(tsnk, cur, policy);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
      InterruptedException {
    final int repeat = 7;
    final int msgs = 10;

    String cfg = "{ mult(" + repeat + ") => counter(\"count\") }";
    EventSink s = FlumeBuilder.buildSink(new ReportTestingContext(), cfg);
    s.open();

    for (int i = 0; i < msgs; i++) {
      Event e = new EventImpl(("" + i).getBytes());
      s.append(e);
    }

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    Assert.assertEquals(msgs * repeat, cnt.getCount());
  }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    final int repeat = 3;
    final int msgs = 4;

    String cfg = "{ benchinject => { mult(" + repeat
        + ") => [console, counter(\"count\")] }}";
    EventSink s = FlumeBuilder.buildSink(new ReportTestingContext(), cfg);
    s.open();

    for (int i = 0; i < msgs; i++) {
      Event e = new EventImpl(("" + i).getBytes());
      s.append(e);
    }
    s.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    // +3 -> start, first, stop
    Assert.assertEquals(msgs * repeat + 3, cnt.getCount());

View Full Code Here
TOP

Related Classes of com.cloudera.flume.core.EventSink

  • com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager
  • com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan
  • com.cloudera.flume.agent.diskfailover.TestDiskFailoverBenchmarking
  • com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco
  • com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager
  • com.cloudera.flume.agent.durability.NaiveFileWALManager
  • com.cloudera.flume.agent.durability.TestConcurrentWALMan
  • com.cloudera.flume.agent.durability.TestNaiveFileWALDeco
  • com.cloudera.flume.agent.durability.TestNaiveFileWALHandles
  • com.cloudera.flume.agent.durability.TestNaiveFileWALManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.