Package com.cloudera.flume.handlers.text.TailSource

Examples of com.cloudera.flume.handlers.text.TailSource.Cursor


    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(10);
    File f = File.createTempFile("appear", ".tmp");
    f.delete();
    f.deleteOnExit();
    Cursor c = new Cursor(q, f);

    assertFalse(c.tailBody()); // attempt to open, nothing there.
    assertFalse(c.tailBody()); // attempt to open, nothing there.
    assertEquals(0, c.lastChannelSize);
    assertEquals(null, c.in);

    FileWriter fw = new FileWriter(f);
    fw.append("No new line");
    fw.close();

    assertTrue(c.tailBody()); // find and load file
    assertEquals(0, c.lastChannelPos);
    assertTrue(null != c.in);

    assertTrue(c.tailBody()); // read but since of EOL, buffer (no progress)
    assertEquals(0, q.size()); // no events since no EOL found
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertFalse(c.tailBody()); // try to read, but in buffer, no progress

    c.flush();
    assertEquals(1, q.size());

    boolean append = true;
    FileWriter fw2 = new FileWriter(f, append);
    fw2.append("more no new line");
    fw2.close();

    assertTrue(c.tailBody()); // open file
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertTrue(c.tailBody()); // read file.
    assertEquals(1, q.size());
    c.flush();
    assertEquals(2, q.size());

  }
View Full Code Here


  public void testCursorTruncate() throws IOException, InterruptedException {
    // normal implementation uses synchronous queue, but we use array blocking
    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(10);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open file the file
    assertEquals(0, c.lastChannelPos);
    assertTrue(null != c.in);

    assertTrue(c.tailBody()); // finish reading the file
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertFalse(c.tailBody()); // attempt to open file again.
    assertEquals(5, q.size()); // should be 5 in queue.

    // truncate the file -- there will be 1 full event and one unproperly closed
    // event.
    RandomAccessFile raf = new RandomAccessFile(f, "rw");
    raf.setLength(10);
    raf.close();

    assertFalse(c.tailBody()); // detect file truncation, no data to read
    assertEquals(5, q.size()); // should be 5 in queue.
    assertEquals(10, c.lastChannelPos);
    assertTrue(null != c.in);

    assertFalse(c.tailBody()); // no data to read
    assertEquals(5, q.size()); // should be 5 in queue.

    appendData(f, 5, 5); // appending data after truncation

    assertTrue(c.tailBody()); // reading appended data
    assertEquals(10, q.size()); // should be 5 + 5 in queue.

    assertFalse(c.tailBody()); // no data to read
    assertEquals(10, q.size()); // should be 5 + 5 in queue.
  }
View Full Code Here

    // normal implementation uses synchronous queue, but we use array blocking
    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f1 = createDataFile(5);
    Cursor c1 = new Cursor(q, f1);

    File f2 = createDataFile(5);
    Cursor c2 = new Cursor(q, f2);

    assertTrue(c1.tailBody()); // open the file
    assertTrue(c2.tailBody()); // open the file
    assertTrue(c1.tailBody()); // open the file
    assertTrue(c2.tailBody()); // open the file
    assertFalse(c1.tailBody()); // read the file
    assertFalse(c2.tailBody()); // no new data.
    assertEquals(10, q.size()); // should be 5 in queue.

    appendData(f1, 5, 5);
    assertTrue(c1.tailBody()); // open the file
    assertFalse(c2.tailBody()); // open the file
    assertEquals(15, q.size()); // should be 5 in queue.

    appendData(f2, 5, 5);
    assertFalse(c1.tailBody()); // open the file
    assertTrue(c2.tailBody()); // open the file
    assertEquals(20, q.size()); // should be 5 in queue.

    assertFalse(c1.tailBody()); // open the file
    assertFalse(c2.tailBody()); // open the file

  }
View Full Code Here

    File f = File.createTempFile("tailexhaust", ".txt");
    f.deleteOnExit();
    File f2 = File.createTempFile("tailexhaust", ".txt");
    f2.deleteOnExit();
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    Cursor c = new Cursor(q, f);

    for (int i = 0; i < 3000; i++) {
      f2.delete();

      appendData(f, i * 5, 5);

      assertTrue(c.tailBody()); // open the file

      f.renameTo(f2); // move the file (should be no problem).

      assertTrue(c.tailBody()); // finish reading the file
      assertEquals(5, q.size()); // should be 5 in queue.

      assertFalse(c.tailBody()); // No more to read
      assertEquals(5, q.size()); // should be 5 in queue.

      q.clear();
    }
    Log.info("file handles didn't leak!");
View Full Code Here

    assertTrue(done.await(10, TimeUnit.SECONDS));

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    assertEquals(count, ctr.getCount());

    Cursor cursor = src.cursors.get(0);
    assertEquals(cursor.lastChannelSize, cursor.lastChannelPos);
  }
View Full Code Here

  public void testCursorPrexisting() throws IOException, InterruptedException {
    // normal implementation uses synchronous queue, but we use array blocking
    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open the file
    assertTrue(c.tailBody()); // read the file
    assertFalse(c.tailBody()); // no new data.
    assertEquals(5, q.size()); // should be 5 in queue.
  }
View Full Code Here

    File f2 = File.createTempFile("move", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open the file

    f.renameTo(f2); // move the file (should be no problem).

    assertTrue(c.tailBody()); // finish reading the file
    assertEquals(5, q.size()); // should be 5 in queue.

    assertFalse(c.tailBody()); // No more to read
    assertEquals(5, q.size()); // should be 5 in queue.

    assertFalse(c.tailBody()); // attempt to open file again.

  }
View Full Code Here

    File f2 = File.createTempFile("move", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open the file

    f.renameTo(f2); // move the file (should be no problem).

    // wait a second to force a new last modified time.
    Clock.sleep(1000);
    appendData(f, 5, 5);

    assertTrue(c.tailBody()); // finish reading the first file
    assertEquals(5, q.size()); // should be 5 in queue.

    // If we had Inode info we could detect if it is a different file, however,
    // we don't. Thus, this tail cannot tell that a new file with the same
    // size is different with available metadata.
    assertTrue(c.tailBody()); // attempt to open file again.
    assertTrue(c.tailBody()); // read 2nd file.
    // This should be 10, but actually results in 5.
    assertEquals(5, q.size()); // should be 5 in queue.
  }
View Full Code Here

    File f2 = File.createTempFile("move", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open the file

    f.renameTo(f2); // move the file (should be no problem).

    // wait a second to force a new last modified time.
    Clock.sleep(1000);
    appendData(f, 5, 6); // should be a new file.

    assertTrue(c.tailBody()); // finish reading the first file
    assertEquals(5, q.size()); // should be 5 in queue.

    assertTrue(c.tailBody()); // notice raflen!= filelen, reset
    assertTrue(c.tailBody()); // read
    assertFalse(c.tailBody()); // no more to read
    assertEquals(11, q.size()); // should be 5 in queue.

    assertFalse(c.tailBody()); // no change this time
    assertEquals(11, q.size());
  }
View Full Code Here

    File f2 = File.createTempFile("move", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open the file

    f.renameTo(f2); // move the file (should be no problem).

    // wait a second to force a new last modified time.
    Clock.sleep(1000);
    appendData(f, 5, 4);

    assertTrue(c.tailBody()); // finish reading the first file
    assertEquals(5, q.size()); // should be 5 in queue.

    assertTrue(c.tailBody()); // notice raflen!= filelen, reset
    assertTrue(c.tailBody()); // attempt to open file again.
    assertFalse(c.tailBody()); // no more to read

    // This should be 10, but actually results in 5.
    assertEquals(9, q.size()); // should be 5 in queue.
  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.handlers.text.TailSource.Cursor

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.