import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.Transaction;
import redis.clients.jedis.Tuple;
import redis.clients.jedis.ZParams;
public class Chapter06 {
public static final void main(String[] args)
throws Exception
{
new Chapter06().run();
}
public void run()
throws InterruptedException, IOException
{
Jedis conn = new Jedis("localhost");
conn.select(15);
testAddUpdateContact(conn);
testAddressBookAutocomplete(conn);
testDistributedLocking(conn);
testCountingSemaphore(conn);
testDelayedTasks(conn);
testMultiRecipientMessaging(conn);
testFileDistribution(conn);
}
public void testAddUpdateContact(Jedis conn) {
System.out.println("\n----- testAddUpdateContact -----");
conn.del("recent:user");
System.out.println("Let's add a few contacts...");
for (int i = 0; i < 10; i++){
addUpdateContact(conn, "user", "contact-" + ((int)Math.floor(i / 3)) + '-' + i);
}
System.out.println("Current recently contacted contacts");
List<String> contacts = conn.lrange("recent:user", 0, -1);
for(String contact : contacts){
System.out.println(" " + contact);
}
assert contacts.size() >= 10;
System.out.println();
System.out.println("Let's pull one of the older ones up to the front");
addUpdateContact(conn, "user", "contact-1-4");
contacts = conn.lrange("recent:user", 0, 2);
System.out.println("New top-3 contacts:");
for(String contact : contacts){
System.out.println(" " + contact);
}
assert "contact-1-4".equals(contacts.get(0));
System.out.println();
System.out.println("Let's remove a contact...");
removeContact(conn, "user", "contact-2-6");
contacts = conn.lrange("recent:user", 0, -1);
System.out.println("New contacts:");
for(String contact : contacts){
System.out.println(" " + contact);
}
assert contacts.size() >= 9;
System.out.println();
System.out.println("And let's finally autocomplete on ");
List<String> all = conn.lrange("recent:user", 0, -1);
contacts = fetchAutocompleteList(conn, "user", "c");
assert all.equals(contacts);
List<String> equiv = new ArrayList<String>();
for (String contact : all){
if (contact.startsWith("contact-2-")){
equiv.add(contact);
}
}
contacts = fetchAutocompleteList(conn, "user", "contact-2-");
Collections.sort(equiv);
Collections.sort(contacts);
assert equiv.equals(contacts);
conn.del("recent:user");
}
public void testAddressBookAutocomplete(Jedis conn) {
System.out.println("\n----- testAddressBookAutocomplete -----");
conn.del("members:test");
System.out.println("the start/end range of 'abc' is: " +
Arrays.toString(findPrefixRange("abc")));
System.out.println();
System.out.println("Let's add a few people to the guild");
for (String name : new String[]{"jeff", "jenny", "jack", "jennifer"}){
joinGuild(conn, "test", name);
}
System.out.println();
System.out.println("now let's try to find users with names starting with 'je':");
Set<String> r = autocompleteOnPrefix(conn, "test", "je");
System.out.println(r);
assert r.size() == 3;
System.out.println("jeff just left to join a different guild...");
leaveGuild(conn, "test", "jeff");
r = autocompleteOnPrefix(conn, "test", "je");
System.out.println(r);
assert r.size() == 2;
conn.del("members:test");
}
public void testDistributedLocking(Jedis conn)
throws InterruptedException
{
System.out.println("\n----- testDistributedLocking -----");
conn.del("lock:testlock");
System.out.println("Getting an initial lock...");
assert acquireLockWithTimeout(conn, "testlock", 1000, 1000) != null;
System.out.println("Got it!");
System.out.println("Trying to get it again without releasing the first one...");
assert acquireLockWithTimeout(conn, "testlock", 10, 1000) == null;
System.out.println("Failed to get it!");
System.out.println();
System.out.println("Waiting for the lock to timeout...");
Thread.sleep(2000);
System.out.println("Getting the lock again...");
String lockId = acquireLockWithTimeout(conn, "testlock", 1000, 1000);
assert lockId != null;
System.out.println("Got it!");
System.out.println("Releasing the lock...");
assert releaseLock(conn, "testlock", lockId);
System.out.println("Released it...");
System.out.println();
System.out.println("Acquiring it again...");
assert acquireLockWithTimeout(conn, "testlock", 1000, 1000) != null;
System.out.println("Got it!");
conn.del("lock:testlock");
}
public void testCountingSemaphore(Jedis conn)
throws InterruptedException
{
System.out.println("\n----- testCountingSemaphore -----");
conn.del("testsem", "testsem:owner", "testsem:counter");
System.out.println("Getting 3 initial semaphores with a limit of 3...");
for (int i = 0; i < 3; i++) {
assert acquireFairSemaphore(conn, "testsem", 3, 1000) != null;
}
System.out.println("Done!");
System.out.println("Getting one more that should fail...");
assert acquireFairSemaphore(conn, "testsem", 3, 1000) == null;
System.out.println("Couldn't get it!");
System.out.println();
System.out.println("Lets's wait for some of them to time out");
Thread.sleep(2000);
System.out.println("Can we get one?");
String id = acquireFairSemaphore(conn, "testsem", 3, 1000);
assert id != null;
System.out.println("Got one!");
System.out.println("Let's release it...");
assert releaseFairSemaphore(conn, "testsem", id);
System.out.println("Released!");
System.out.println();
System.out.println("And let's make sure we can get 3 more!");
for (int i = 0; i < 3; i++) {
assert acquireFairSemaphore(conn, "testsem", 3, 1000) != null;
}
System.out.println("We got them!");
conn.del("testsem", "testsem:owner", "testsem:counter");
}
public void testDelayedTasks(Jedis conn)
throws InterruptedException
{
System.out.println("\n----- testDelayedTasks -----");
conn.del("queue:tqueue", "delayed:");
System.out.println("Let's start some regular and delayed tasks...");
for (long delay : new long[]{0, 500, 0, 1500}){
assert executeLater(conn, "tqueue", "testfn", new ArrayList<String>(), delay) != null;
}
long r = conn.llen("queue:tqueue");
System.out.println("How many non-delayed tasks are there (should be 2)? " + r);
assert r == 2;
System.out.println();
System.out.println("Let's start up a thread to bring those delayed tasks back...");
PollQueueThread thread = new PollQueueThread();
thread.start();
System.out.println("Started.");
System.out.println("Let's wait for those tasks to be prepared...");
Thread.sleep(2000);
thread.quit();
thread.join();
r = conn.llen("queue:tqueue");
System.out.println("Waiting is over, how many tasks do we have (should be 4)? " + r);
assert r == 4;
conn.del("queue:tqueue", "delayed:");
}
public void testMultiRecipientMessaging(Jedis conn) {
System.out.println("\n----- testMultiRecipientMessaging -----");
conn.del("ids:chat:", "msgs:1", "ids:1", "seen:joe", "seen:jeff", "seen:jenny");
System.out.println("Let's create a new chat session with some recipients...");
Set<String> recipients = new HashSet<String>();
recipients.add("jeff");
recipients.add("jenny");
String chatId = createChat(conn, "joe", recipients, "message 1");
System.out.println("Now let's send a few messages...");
for (int i = 2; i < 5; i++){
sendMessage(conn, chatId, "joe", "message " + i);
}
System.out.println();
System.out.println("And let's get the messages that are waiting for jeff and jenny...");
List<ChatMessages> r1 = fetchPendingMessages(conn, "jeff");
List<ChatMessages> r2 = fetchPendingMessages(conn, "jenny");
System.out.println("They are the same? " + r1.equals(r2));
assert r1.equals(r2);
System.out.println("Those messages are:");
for(ChatMessages chat : r1){
System.out.println(" chatId: " + chat.chatId);
System.out.println(" messages:");
for(Map<String,Object> message : chat.messages){
System.out.println(" " + message);
}
}
conn.del("ids:chat:", "msgs:1", "ids:1", "seen:joe", "seen:jeff", "seen:jenny");
}
public void testFileDistribution(Jedis conn)
throws InterruptedException, IOException
{
System.out.println("\n----- testFileDistribution -----");
String[] keys = conn.keys("test:*").toArray(new String[0]);
if (keys.length > 0){
conn.del(keys);
}
conn.del(
"msgs:test:",
"seen:0",
"seen:source",
"ids:test:",
"chat:test:");
System.out.println("Creating some temporary 'log' files...");
File f1 = File.createTempFile("temp_redis_1_", ".txt");
f1.deleteOnExit();
Writer writer = new FileWriter(f1);
writer.write("one line\n");
writer.close();
File f2 = File.createTempFile("temp_redis_2_", ".txt");
f2.deleteOnExit();
writer = new FileWriter(f2);
for (int i = 0; i < 100; i++){
writer.write("many lines " + i + '\n');
}
writer.close();
File f3 = File.createTempFile("temp_redis_3_", ".txt.gz");
f3.deleteOnExit();
writer = new OutputStreamWriter(
new GZIPOutputStream(
new FileOutputStream(f3)));
Random random = new Random();
for (int i = 0; i < 1000; i++){
writer.write("random line " + Long.toHexString(random.nextLong()) + '\n');
}
writer.close();
long size = f3.length();
System.out.println("Done.");
System.out.println();
System.out.println("Starting up a thread to copy logs to redis...");
File path = f1.getParentFile();
CopyLogsThread thread = new CopyLogsThread(path, "test:", 1, size);
thread.start();
System.out.println("Let's pause to let some logs get copied to Redis...");
Thread.sleep(250);
System.out.println();
System.out.println("Okay, the logs should be ready. Let's process them!");
System.out.println("Files should have 1, 100, and 1000 lines");
TestCallback callback = new TestCallback();
processLogsFromRedis(conn, "0", callback);
System.out.println(java.util.Arrays.toString(callback.counts.toArray(new Integer[0])));
assert callback.counts.get(0) == 1;
assert callback.counts.get(1) == 100;
assert callback.counts.get(2) == 1000;
System.out.println();
System.out.println("Let's wait for the copy thread to finish cleaning up...");
thread.join();
System.out.println("Done cleaning out Redis!");
keys = conn.keys("test:*").toArray(new String[0]);
if (keys.length > 0){
conn.del(keys);
}
conn.del(
"msgs:test:",
"seen:0",
"seen:source",
"ids:test:",
"chat:test:");
}
public class TestCallback
implements Callback
{
private int index;
public List<Integer> counts = new ArrayList<Integer>();
public void callback(String line){
if (line == null){
index++;
return;
}
while (counts.size() == index){
counts.add(0);
}
counts.set(index, counts.get(index) + 1);
}
}
public void addUpdateContact(Jedis conn, String user, String contact) {
String acList = "recent:" + user;
Transaction trans = conn.multi();
trans.lrem(acList, 0, contact);
trans.lpush(acList, contact);
trans.ltrim(acList, 0, 99);
trans.exec();
}
public void removeContact(Jedis conn, String user, String contact) {
conn.lrem("recent:" + user, 0, contact);
}
public List<String> fetchAutocompleteList(Jedis conn, String user, String prefix) {
List<String> candidates = conn.lrange("recent:" + user, 0, -1);
List<String> matches = new ArrayList<String>();
for (String candidate : candidates) {
if (candidate.toLowerCase().startsWith(prefix)){
matches.add(candidate);
}
}
return matches;
}
private static final String VALID_CHARACTERS = "`abcdefghijklmnopqrstuvwxyz{";
public String[] findPrefixRange(String prefix) {
int posn = VALID_CHARACTERS.indexOf(prefix.charAt(prefix.length() - 1));
char suffix = VALID_CHARACTERS.charAt(posn > 0 ? posn - 1 : 0);
String start = prefix.substring(0, prefix.length() - 1) + suffix + '{';
String end = prefix + '{';
return new String[]{start, end};
}
public void joinGuild(Jedis conn, String guild, String user) {
conn.zadd("members:" + guild, 0, user);
}
public void leaveGuild(Jedis conn, String guild, String user) {
conn.zrem("members:" + guild, user);
}
@SuppressWarnings("unchecked")
public Set<String> autocompleteOnPrefix(Jedis conn, String guild, String prefix) {
String[] range = findPrefixRange(prefix);
String start = range[0];
String end = range[1];
String identifier = UUID.randomUUID().toString();
start += identifier;
end += identifier;
String zsetName = "members:" + guild;
conn.zadd(zsetName, 0, start);
conn.zadd(zsetName, 0, end);
Set<String> items = null;
while (true){
conn.watch(zsetName);
int sindex = conn.zrank(zsetName, start).intValue();
int eindex = conn.zrank(zsetName, end).intValue();
int erange = Math.min(sindex + 9, eindex - 2);
Transaction trans = conn.multi();
trans.zrem(zsetName, start);
trans.zrem(zsetName, end);
trans.zrange(zsetName, sindex, erange);
List<Object> results = trans.exec();
if (results != null){
items = (Set<String>)results.get(results.size() - 1);
break;
}
}
for (Iterator<String> iterator = items.iterator(); iterator.hasNext(); ){
if (iterator.next().indexOf('{') != -1){
iterator.remove();
}
}
return items;
}
public String acquireLock(Jedis conn, String lockName) {
return acquireLock(conn, lockName, 10000);
}
public String acquireLock(Jedis conn, String lockName, long acquireTimeout){
String identifier = UUID.randomUUID().toString();
long end = System.currentTimeMillis() + acquireTimeout;
while (System.currentTimeMillis() < end){
if (conn.setnx("lock:" + lockName, identifier) == 1){
return identifier;
}
try {
Thread.sleep(1);
}catch(InterruptedException ie){
Thread.currentThread().interrupt();
}
}
return null;
}
public String acquireLockWithTimeout(
Jedis conn, String lockName, long acquireTimeout, long lockTimeout)
{
String identifier = UUID.randomUUID().toString();
String lockKey = "lock:" + lockName;
int lockExpire = (int)(lockTimeout / 1000);
long end = System.currentTimeMillis() + acquireTimeout;
while (System.currentTimeMillis() < end) {
if (conn.setnx(lockKey, identifier) == 1){
conn.expire(lockKey, lockExpire);
return identifier;
}
if (conn.ttl(lockKey) == -1) {
conn.expire(lockKey, lockExpire);
}
try {
Thread.sleep(1);
}catch(InterruptedException ie){
Thread.currentThread().interrupt();
}
}
// null indicates that the lock was not acquired
return null;
}
public boolean releaseLock(Jedis conn, String lockName, String identifier) {
String lockKey = "lock:" + lockName;
while (true){
conn.watch(lockKey);
if (identifier.equals(conn.get(lockKey))){
Transaction trans = conn.multi();
trans.del(lockKey);
List<Object> results = trans.exec();
if (results == null){
continue;
}
return true;
}
conn.unwatch();
break;
}
return false;
}
public String acquireFairSemaphore(
Jedis conn, String semname, int limit, long timeout)
{
String identifier = UUID.randomUUID().toString();
String czset = semname + ":owner";
String ctr = semname + ":counter";
long now = System.currentTimeMillis();
Transaction trans = conn.multi();
trans.zremrangeByScore(
semname.getBytes(),
"-inf".getBytes(),
String.valueOf(now - timeout).getBytes());
ZParams params = new ZParams();
params.weights(1, 0);
trans.zinterstore(czset, params, czset, semname);
trans.incr(ctr);
List<Object> results = trans.exec();
int counter = ((Long)results.get(results.size() - 1)).intValue();
trans = conn.multi();
trans.zadd(semname, now, identifier);
trans.zadd(czset, counter, identifier);
trans.zrank(czset, identifier);
results = trans.exec();
int result = ((Long)results.get(results.size() - 1)).intValue();
if (result < limit){
return identifier;
}
trans = conn.multi();
trans.zrem(semname, identifier);
trans.zrem(czset, identifier);
trans.exec();
return null;
}
public boolean releaseFairSemaphore(
Jedis conn, String semname, String identifier)
{
Transaction trans = conn.multi();
trans.zrem(semname, identifier);
trans.zrem(semname + ":owner", identifier);
List<Object> results = trans.exec();
return (Long)results.get(results.size() - 1) == 1;
}
public String executeLater(
Jedis conn, String queue, String name, List<String> args, long delay)
{
Gson gson = new Gson();
String identifier = UUID.randomUUID().toString();
String itemArgs = gson.toJson(args);
String item = gson.toJson(new String[]{identifier, queue, name, itemArgs});
if (delay > 0){
conn.zadd("delayed:", System.currentTimeMillis() + delay, item);
} else {
conn.rpush("queue:" + queue, item);
}
return identifier;
}
public String createChat(Jedis conn, String sender, Set<String> recipients, String message) {
String chatId = String.valueOf(conn.incr("ids:chat:"));
return createChat(conn, sender, recipients, message, chatId);
}
public String createChat(
Jedis conn, String sender, Set<String> recipients, String message, String chatId)
{
recipients.add(sender);
Transaction trans = conn.multi();
for (String recipient : recipients){
trans.zadd("chat:" + chatId, 0, recipient);
trans.zadd("seen:" + recipient, 0, chatId);
}
trans.exec();
return sendMessage(conn, chatId, sender, message);
}
public String sendMessage(Jedis conn, String chatId, String sender, String message) {
String identifier = acquireLock(conn, "chat:" + chatId);
if (identifier == null){
throw new RuntimeException("Couldn't get the lock");
}
try {
long messageId = conn.incr("ids:" + chatId);
HashMap<String,Object> values = new HashMap<String,Object>();
values.put("id", messageId);
values.put("ts", System.currentTimeMillis());
values.put("sender", sender);
values.put("message", message);
String packed = new Gson().toJson(values);
conn.zadd("msgs:" + chatId, messageId, packed);
}finally{
releaseLock(conn, "chat:" + chatId, identifier);
}
return chatId;
}
@SuppressWarnings("unchecked")
public List<ChatMessages> fetchPendingMessages(Jedis conn, String recipient) {
Set<Tuple> seenSet = conn.zrangeWithScores("seen:" + recipient, 0, -1);
List<Tuple> seenList = new ArrayList<Tuple>(seenSet);
Transaction trans = conn.multi();
for (Tuple tuple : seenList){
String chatId = tuple.getElement();
int seenId = (int)tuple.getScore();
trans.zrangeByScore("msgs:" + chatId, String.valueOf(seenId + 1), "inf");
}
List<Object> results = trans.exec();
Gson gson = new Gson();
Iterator<Tuple> seenIterator = seenList.iterator();
Iterator<Object> resultsIterator = results.iterator();
List<ChatMessages> chatMessages = new ArrayList<ChatMessages>();
List<Object[]> seenUpdates = new ArrayList<Object[]>();
List<Object[]> msgRemoves = new ArrayList<Object[]>();
while (seenIterator.hasNext()){
Tuple seen = seenIterator.next();
Set<String> messageStrings = (Set<String>)resultsIterator.next();
if (messageStrings.size() == 0){
continue;
}
int seenId = 0;
String chatId = seen.getElement();
List<Map<String,Object>> messages = new ArrayList<Map<String,Object>>();
for (String messageJson : messageStrings){
Map<String,Object> message = (Map<String,Object>)gson.fromJson(
messageJson, new TypeToken<Map<String,Object>>(){}.getType());
int messageId = ((Double)message.get("id")).intValue();
if (messageId > seenId){
seenId = messageId;
}
message.put("id", messageId);
messages.add(message);
}
conn.zadd("chat:" + chatId, seenId, recipient);
seenUpdates.add(new Object[]{"seen:" + recipient, seenId, chatId});
Set<Tuple> minIdSet = conn.zrangeWithScores("chat:" + chatId, 0, 0);
if (minIdSet.size() > 0){
msgRemoves.add(new Object[]{
"msgs:" + chatId, minIdSet.iterator().next().getScore()});
}
chatMessages.add(new ChatMessages(chatId, messages));
}
trans = conn.multi();
for (Object[] seenUpdate : seenUpdates){
trans.zadd(
(String)seenUpdate[0],
(Integer)seenUpdate[1],
(String)seenUpdate[2]);
}
for (Object[] msgRemove : msgRemoves){
trans.zremrangeByScore(
(String)msgRemove[0], 0, ((Double)msgRemove[1]).intValue());
}
trans.exec();
return chatMessages;
}
public void processLogsFromRedis(Jedis conn, String id, Callback callback)
throws InterruptedException, IOException
{
while (true){
List<ChatMessages> fdata = fetchPendingMessages(conn, id);
for (ChatMessages messages : fdata){
for (Map<String,Object> message : messages.messages){
String logFile = (String)message.get("message");
if (":done".equals(logFile)){
return;
}
if (logFile == null || logFile.length() == 0){
continue;
}
InputStream in = new RedisInputStream(
conn, messages.chatId + logFile);
if (logFile.endsWith(".gz")){
in = new GZIPInputStream(in);
}
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
try{
String line = null;
while ((line = reader.readLine()) != null){
callback.callback(line);
}
callback.callback(null);
}finally{
reader.close();
}
conn.incr(messages.chatId + logFile + ":done");
}
}
if (fdata.size() == 0){
Thread.sleep(100);
}
}
}
public class RedisInputStream
extends InputStream
{
private Jedis conn;
private String key;
private int pos;
public RedisInputStream(Jedis conn, String key){
this.conn = conn;
this.key = key;
}
@Override
public int available()
throws IOException
{
long len = conn.strlen(key);
return (int)(len - pos);
}
@Override
public int read()
throws IOException
{
byte[] block = conn.substr(key.getBytes(), pos, pos);
if (block == null || block.length == 0){
return -1;
}
pos++;
return (int)(block[0] & 0xff);
}
@Override
public int read(byte[] buf, int off, int len)
throws IOException
{
byte[] block = conn.substr(key.getBytes(), pos, pos + (len - off - 1));
if (block == null || block.length == 0){
return -1;
}
System.arraycopy(block, 0, buf, off, block.length);
pos += block.length;
return block.length;
}
@Override
public void close() {
// no-op
}
}
public interface Callback {
void callback(String line);
}
public class ChatMessages
{
public String chatId;
public List<Map<String,Object>> messages;
public ChatMessages(String chatId, List<Map<String,Object>> messages){
this.chatId = chatId;
this.messages = messages;
}
public boolean equals(Object other){
if (!(other instanceof ChatMessages)){
return false;
}
ChatMessages otherCm = (ChatMessages)other;
return chatId.equals(otherCm.chatId) &&
messages.equals(otherCm.messages);
}
}
public class PollQueueThread
extends Thread
{
private Jedis conn;
private boolean quit;
private Gson gson = new Gson();
public PollQueueThread(){
this.conn = new Jedis("localhost");
this.conn.select(15);
}
public void quit() {
quit = true;
}
public void run() {
while (!quit){
Set<Tuple> items = conn.zrangeWithScores("delayed:", 0, 0);
Tuple item = items.size() > 0 ? items.iterator().next() : null;
if (item == null || item.getScore() > System.currentTimeMillis()) {
try{
sleep(10);
}catch(InterruptedException ie){
Thread.interrupted();
}
continue;
}
String json = item.getElement();
String[] values = gson.fromJson(json, String[].class);
String identifier = values[0];
String queue = values[1];
String locked = acquireLock(conn, identifier);
if (locked == null){
continue;
}
if (conn.zrem("delayed:", json) == 1){
conn.rpush("queue:" + queue, json);
}
releaseLock(conn, identifier, locked);
}
}
}
public class CopyLogsThread
extends Thread
{
private Jedis conn;
private File path;
private String channel;
private int count;
private long limit;
public CopyLogsThread(File path, String channel, int count, long limit) {
this.conn = new Jedis("localhost");
this.conn.select(15);
this.path = path;
this.channel = channel;
this.count = count;
this.limit = limit;
}
public void run() {
Deque<File> waiting = new ArrayDeque<File>();
long bytesInRedis = 0;
Set<String> recipients= new HashSet<String>();
for (int i = 0; i < count; i++){
recipients.add(String.valueOf(i));
}
createChat(conn, "source", recipients, "", channel);
File[] logFiles = path.listFiles(new FilenameFilter(){
public boolean accept(File dir, String name){
return name.startsWith("temp_redis");
}
});
Arrays.sort(logFiles);
for (File logFile : logFiles){
long fsize = logFile.length();
while ((bytesInRedis + fsize) > limit){
long cleaned = clean(waiting, count);
if (cleaned != 0){
bytesInRedis -= cleaned;
}else{
try{
sleep(250);
}catch(InterruptedException ie){
Thread.interrupted();
}
}
}
BufferedInputStream in = null;
try{
in = new BufferedInputStream(new FileInputStream(logFile));
int read = 0;
byte[] buffer = new byte[8192];
while ((read = in.read(buffer, 0, buffer.length)) != -1){
if (buffer.length != read){
byte[] bytes = new byte[read];
System.arraycopy(buffer, 0, bytes, 0, read);
conn.append((channel + logFile).getBytes(), bytes);
}else{
conn.append((channel + logFile).getBytes(), buffer);
}
}
}catch(IOException ioe){
ioe.printStackTrace();
throw new RuntimeException(ioe);
}finally{
try{
in.close();
}catch(Exception ignore){
}
}
sendMessage(conn, channel, "source", logFile.toString());
bytesInRedis += fsize;
waiting.addLast(logFile);
}
sendMessage(conn, channel, "source", ":done");
while (waiting.size() > 0){
long cleaned = clean(waiting, count);
if (cleaned != 0){
bytesInRedis -= cleaned;
}else{
try{
sleep(250);
}catch(InterruptedException ie){
Thread.interrupted();
}
}
}
}
private long clean(Deque<File> waiting, int count) {
if (waiting.size() == 0){
return 0;
}
File w0 = waiting.getFirst();
if (String.valueOf(count).equals(conn.get(channel + w0 + ":done"))){
conn.del(channel + w0, channel + w0 + ":done");
return waiting.removeFirst().length();
}
return 0;
}
}
}