Package com.cloudera.flume.handlers.text.TailSource

Examples of com.cloudera.flume.handlers.text.TailSource.Cursor


    File f2 = File.createTempFile("move", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open the file

    f.renameTo(f2); // move the file (should be no problem).

    assertTrue(c.tailBody()); // finish reading the first file
    assertEquals(5, q.size()); // should be 5 in queue.

    assertFalse(c.tailBody()); // attempt to file again. (not there, no
    // progress)

    // wait a second to force a new last modified time.
    Clock.sleep(1000);
    appendData(f, 5, 5);

    assertTrue(c.tailBody()); // open the new file

    assertTrue(c.tailBody()); // read new file
    assertFalse(c.tailBody()); // fails this time
    assertEquals(10, q.size());
  }
View Full Code Here


    File f2 = File.createTempFile("move", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open the file
    assertTrue(c.tailBody()); // finish reading the first file
    assertEquals(5, q.size()); // should be 5 in queue.

    assertFalse(c.tailBody()); // attempt to file again. (not there)
    assertFalse(c.tailBody()); // attempt to file again. (not there)

    // wait a second to force a new last modified time.
    f.renameTo(f2); // move the file (should be no problem).
    Clock.sleep(1000);
    appendData(f, 5, 5);

    assertTrue(c.tailBody()); // notice new mod time and reset
    assertTrue(c.tailBody()); // open the new file
    assertTrue(c.tailBody()); // read new file
    assertFalse(c.tailBody()); // fails this time
    assertEquals(10, q.size());
  }
View Full Code Here

    // normal implementation uses synchronous queue, but we use array blocking
    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(10);
    File f = createDataFile(5);
    f.deleteOnExit();
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open the file

    assertTrue(c.tailBody()); // finish reading the file

    appendData(f, 5, 5);

    assertTrue(c.tailBody()); // attempt to open file again.
    assertEquals(10, q.size()); // should be 5 in queue.
  }
View Full Code Here

    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(10);
    File f = File.createTempFile("appear", ".tmp");
    f.delete();
    f.deleteOnExit();
    Cursor c = new Cursor(q, f);

    assertFalse(c.tailBody()); // attempt to open, nothing there.
    assertFalse(c.tailBody()); // attempt to open, nothing there.
    assertEquals(0, c.lastChannelSize);
    assertEquals(null, c.in);

    appendData(f, 0, 5);
    assertTrue(c.tailBody()); // finish reading the file
    assertEquals(0, c.lastChannelPos);
    assertTrue(null != c.in);

    assertTrue(c.tailBody()); // finish reading the file
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertFalse(c.tailBody()); // attempt to open file again.
    assertEquals(5, q.size()); // should be 5 in queue.
  }
View Full Code Here

    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(10);
    File f = File.createTempFile("appear", ".tmp");
    f.delete();
    f.deleteOnExit();
    Cursor c = new Cursor(q, f);

    assertFalse(c.tailBody()); // attempt to open, nothing there.
    assertFalse(c.tailBody()); // attempt to open, nothing there.
    assertEquals(0, c.lastChannelSize);
    assertEquals(null, c.in);

    FileWriter fw = new FileWriter(f);
    fw.append("No new line");
    fw.close();

    assertTrue(c.tailBody()); // find and load file
    assertEquals(0, c.lastChannelPos);
    assertTrue(null != c.in);

    assertTrue(c.tailBody()); // read but since of EOL, buffer (no progress)
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertFalse(c.tailBody()); // try to read, but in buffer, no progress

    c.flush();
    assertEquals(1, q.size());

    boolean append = true;
    FileWriter fw2 = new FileWriter(f, append);
    fw2.append("more no new line");
    fw2.close();

    assertTrue(c.tailBody()); // open file
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertTrue(c.tailBody()); // read file.
    assertEquals(1, q.size());
    c.flush();
    assertEquals(2, q.size());

  }
View Full Code Here

  public void testCursorTruncate() throws IOException, InterruptedException {
    // normal implementation uses synchronous queue, but we use array blocking
    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(10);
    File f = createDataFile(5);
    Cursor c = new Cursor(q, f);

    assertTrue(c.tailBody()); // open file the file
    assertEquals(0, c.lastChannelPos);
    assertTrue(null != c.in);

    assertTrue(c.tailBody()); // finish reading the file
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertFalse(c.tailBody()); // attempt to open file again.
    assertEquals(5, q.size()); // should be 5 in queue.

    // truncate the file -- there will be 1 full event and one unproperly closed
    // event.
    RandomAccessFile raf = new RandomAccessFile(f, "rw");
    raf.setLength(10);
    raf.close();

    // TODO (jon): When a file gets truncated, because we don't have an
    // inode we don't know if this is a new file or a truncated version of the
    // original file. This is not a fundamental flaw -- safe assumption here is
    // that it is a new file and according to our semantics, we are just
    // duplicating information. NOTE: This is another place where this tail
    // differs from the semantics of gnu-tail.

    assertTrue(c.tailBody()); // reset changed file
    assertEquals(0, c.lastChannelPos);
    assertEquals(null, c.in);

    assertTrue(c.tailBody()); // re-opens length changed file
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertTrue(c.tailBody()); // finish reading the file
    assertTrue(0 != c.lastChannelSize);
    assertTrue(null != c.in);

    assertFalse(c.tailBody()); // attempt to open file again.
    assertEquals(6, q.size()); // should be 5 in queue.

    c.flush();
    assertTrue(c.tailBody()); // attempt to open file again.
    assertEquals(7, q.size()); // should be 5 in queue.

  }
View Full Code Here

    // normal implementation uses synchronous queue, but we use array blocking
    // queue for single threaded testing
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    File f1 = createDataFile(5);
    Cursor c1 = new Cursor(q, f1);

    File f2 = createDataFile(5);
    Cursor c2 = new Cursor(q, f2);

    assertTrue(c1.tailBody()); // open the file
    assertTrue(c2.tailBody()); // open the file
    assertTrue(c1.tailBody()); // open the file
    assertTrue(c2.tailBody()); // open the file
    assertFalse(c1.tailBody()); // read the file
    assertFalse(c2.tailBody()); // no new data.
    assertEquals(10, q.size()); // should be 5 in queue.

    appendData(f1, 5, 5);
    assertTrue(c1.tailBody()); // open the file
    assertFalse(c2.tailBody()); // open the file
    assertEquals(15, q.size()); // should be 5 in queue.

    appendData(f2, 5, 5);
    assertFalse(c1.tailBody()); // open the file
    assertTrue(c2.tailBody()); // open the file
    assertEquals(20, q.size()); // should be 5 in queue.

    assertFalse(c1.tailBody()); // open the file
    assertFalse(c2.tailBody()); // open the file

  }
View Full Code Here

    File f = File.createTempFile("tailexhaust", ".txt");
    f.deleteOnExit();
    File f2 = File.createTempFile("tailexhaust", ".txt");
    f2.deleteOnExit();
    BlockingQueue<Event> q = new ArrayBlockingQueue<Event>(100);
    Cursor c = new Cursor(q, f);

    for (int i = 0; i < 3000; i++) {
      f2.delete();

      appendData(f, i * 5, 5);

      assertTrue(c.tailBody()); // open the file

      f.renameTo(f2); // move the file (should be no problem).

      assertTrue(c.tailBody()); // finish reading the file
      assertEquals(5, q.size()); // should be 5 in queue.

      assertFalse(c.tailBody()); // No more to read
      assertEquals(5, q.size()); // should be 5 in queue.

      q.clear();
    }
    Log.info("file handles didn't leak!");
View Full Code Here

    assertTrue(done.await(10, TimeUnit.SECONDS));

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    assertEquals(count, ctr.getCount());

    Cursor cursor = src.cursors.get(0);
    assertEquals(cursor.lastChannelSize, cursor.lastChannelPos);
  }
View Full Code Here

                + "into subdirectory " + f);
            return;
          }
          if (f.lastModified() >= startFromDateInEpoch) {
            LOG.info("File: " + f + " lastModifiedAt: " + f.lastModified() + " is younger than: " + startFromDateInEpoch + " adding to list");
            Cursor c = new Cursor(tail.sync, f);
            curmap.put(f.getName(), c);
            tail.addCursor(c);
            filesAdded.incrementAndGet();
          } else {
            LOG.info("File: " + f + " lastModifiedAt: " + f.lastModified() + " is older than: " + startFromDateInEpoch + " not adding to list");
          }
        }

        @Override
        public void fileDeleted(File f) {
          LOG.info("removed file " + f);
          Cursor c = curmap.remove(f.getName());
          tail.removeCursor(c);
          filesDeleted.incrementAndGet();
        }

      });
View Full Code Here

TOP

Related Classes of com.cloudera.flume.handlers.text.TailSource.Cursor

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.