/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.UUID;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.archive.bdb.BdbModule;
import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.archive.util.TmpDirTestCase;
/**
* Tests that {@link BdbCookieStore} matches behavior of
* {@link BasicCookieStore}.
*
* @contributor nlevitt
*/
public class CookieStoreTest extends TmpDirTestCase {
private static Logger logger = Logger.getLogger(CookieStoreTest.class.getName());
protected BdbModule bdb;
protected BdbCookieStore bdbCookieStore;
protected BasicCookieStore basicCookieStore;
protected BdbModule bdb() throws IOException {
if (bdb == null) {
ConfigPath basePath = new ConfigPath("testBase",
getTmpDir().getAbsolutePath());
ConfigPath bdbDir = new ConfigPath("bdb", "bdb");
bdbDir.setBase(basePath);
FileUtils.deleteDirectory(bdbDir.getFile());
bdb = new BdbModule();
bdb.setDir(bdbDir);
bdb.start();
logger.info("created " + bdb);
}
return bdb;
}
protected AbstractCookieStore bdbCookieStore() throws IOException {
if (bdbCookieStore == null) {
bdbCookieStore = new BdbCookieStore();
ConfigPath basePath = new ConfigPath("testBase",
getTmpDir().getAbsolutePath());
ConfigFile cookiesSaveFile = new ConfigFile("cookiesSaveFile", "cookies.txt");
cookiesSaveFile.setBase(basePath);
bdbCookieStore.setCookiesSaveFile(cookiesSaveFile);
bdbCookieStore.setBdbModule(bdb());
bdbCookieStore.start();
}
return bdbCookieStore;
}
protected BasicCookieStore basicCookieStore() {
if (basicCookieStore == null) {
basicCookieStore = new BasicCookieStore();
}
return basicCookieStore;
}
@Override
protected void tearDown() throws Exception {
super.tearDown();
bdb.close();
}
public void testBasics() throws IOException {
bdbCookieStore().clear();
basicCookieStore().clear();
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
}
public void testSimpleReplace() throws IOException {
bdbCookieStore().clear();
basicCookieStore().clear();
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// should replace existing cookie
cookie = new BasicClientCookie("name1", "value2");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
}
public void testDomains() throws IOException {
bdbCookieStore().clear();
basicCookieStore().clear();
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
cookie.setDomain("example.org");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// add a 2nd cookie, same name, different domain
cookie = new BasicClientCookie("name1", "value2");
cookie.setDomain("example.com");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// add a 3rd cookie, same name, different domain
cookie = new BasicClientCookie("name1", "value3");
cookie.setDomain("foo.example.com");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// replace 1st cookie
cookie = new BasicClientCookie("name1", "value4");
cookie.setDomain("example.org");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// replace 2nd cookie, case-insensitive domain
cookie = new BasicClientCookie("name1", "value5");
cookie.setDomain("eXaMpLe.CoM");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
}
public void testPaths() throws IOException {
bdbCookieStore().clear();
basicCookieStore().clear();
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// replace 1st cookie, with explicit path "/", which is the implied path if not specified
cookie = new BasicClientCookie("name1", "value2");
cookie.setPath("/");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertEquals(1, basicCookieStore().getCookies().size());
assertEquals(cookie, new ArrayList<Cookie>(basicCookieStore().getCookies()).get(0));
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// add a 2nd cookie at a subpath
cookie = new BasicClientCookie("name1", "value3");
cookie.setPath("/path1");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertEquals(2, basicCookieStore().getCookies().size());
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// add a 3rd cookie at a subpath
cookie = new BasicClientCookie("name1", "value4");
cookie.setPath("/path2");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertEquals(3, basicCookieStore().getCookies().size());
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// replace 2nd cookie
cookie = new BasicClientCookie("name1", "value5");
cookie.setPath("/path1");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertEquals(3, basicCookieStore().getCookies().size());
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// add 4th cookie at previously used path
cookie = new BasicClientCookie("name2", "value6");
cookie.setPath("/path1");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertEquals(4, basicCookieStore().getCookies().size());
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
// add 5th cookie at different path (case sensitive)
cookie = new BasicClientCookie("name1", "value7");
cookie.setPath("/pAtH1");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
assertEquals(5, basicCookieStore().getCookies().size());
assertCookieStoresEquivalent(basicCookieStore(), bdbCookieStore());
}
/*
* saveCookies() expects non-null domain, and real world cookies always have
* a domain. And only test attributes that are saved in cookies.txt.
*/
public void testSaveLoadCookies() throws IOException {
bdbCookieStore().clear();
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
cookie.setDomain("example.com");
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name2", "value2");
cookie.setDomain("example.com");
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name3", "value3");
cookie.setDomain("example.com");
cookie.setSecure(true);
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name4", "value4");
cookie.setDomain("example.org");
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name5", "value5");
cookie.setDomain("example.com");
// make sure date is in the future so cookie doesn't expire, and has no
// millisecond value, so save/load loses no info
long someFutureDateMs = ((System.currentTimeMillis() + 9999999999l)/1000l)*1000l;
cookie.setExpiryDate(new Date(someFutureDateMs));
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name6", "value6");
cookie.setDomain("example.com");
cookie.setPath("/path1");
bdbCookieStore().addCookie(cookie);
List<Cookie> cookiesBefore = new ArrayList<Cookie>(bdbCookieStore().getCookies());
assertEquals(6, cookiesBefore.size());
bdbCookieStore().saveCookies();
bdbCookieStore().clear();
assertEquals(0, bdbCookieStore().getCookies().size());
bdbCookieStore().loadCookies((ConfigFile) bdbCookieStore().getCookiesSaveFile());
List<Cookie> cookiesAfter = new ArrayList<Cookie>(bdbCookieStore().getCookies());
assertEquals(6, cookiesAfter.size());
logger.info("before: " + cookiesBefore);
logger.info(" after: " + cookiesAfter);
assertCookieListsEquivalent(cookiesBefore, cookiesAfter);
}
public void testConcurrentLoad() throws IOException, InterruptedException {
bdbCookieStore().clear();
basicCookieStore().clear();
final Random rand = new Random();
Runnable runnable = new Runnable() {
@Override
public void run() {
try {
while (!Thread.interrupted()) {
BasicClientCookie cookie = new BasicClientCookie(UUID.randomUUID().toString(), UUID.randomUUID().toString());
cookie.setDomain("d" + rand.nextInt(10) + ".example.com");
bdbCookieStore().addCookie(cookie);
basicCookieStore().addCookie(cookie);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
Thread[] threads = new Thread[200];
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(runnable);
threads[i].setName("cookie-load-test-" + i);
threads[i].start();
}
Thread.sleep(5000);
for (int i = 0; i < threads.length; i++) {
threads[i].interrupt();
}
for (int i = 0; i < threads.length; i++) {
threads[i].join();
}
List<Cookie> bdbCookieList = bdbCookieStore().getCookies();
assertTrue(bdbCookieList.size() > 3000);
assertCookieListsEquivalent(bdbCookieList, basicCookieStore().getCookies());
}
protected void assertCookieListsEquivalent(List<Cookie> list1,
List<Cookie> list2) {
Comparator<Cookie> comparator = new Comparator<Cookie>() {
@Override
public int compare(Cookie o1, Cookie o2) {
return o1.toString().compareTo(o2.toString());
}
};
ArrayList<Cookie> sorted1 = new ArrayList<Cookie>(list1);
Collections.sort(sorted1, comparator);
ArrayList<Cookie> sorted2 = new ArrayList<Cookie>(list2);
Collections.sort(sorted2, comparator);
assertEquals(list1.size(), list2.size());
assertEquals(sorted1.size(), sorted2.size());
Iterator<Cookie> iter1 = sorted1.iterator();
Iterator<Cookie> iter2 = sorted2.iterator();
for (int i = 0; i < sorted1.size(); i++) {
Cookie c1 = iter1.next();
Cookie c2 = iter2.next();
assertCookiesIdentical(c1, c2);
}
}
protected void assertCookieStoresEquivalent(BasicCookieStore simple, AbstractCookieStore bdb) {
assertCookieListsEquivalent(simple.getCookies(), bdb.getCookies());
}
protected void assertCookiesIdentical(Cookie c1, Cookie c2) {
assertEquals(c1.getComment(), c2.getComment());
assertEquals(c1.getCommentURL(), c2.getCommentURL());
assertEquals(c1.getDomain(), c2.getDomain());
assertEquals(c1.getName(), c2.getName());
String p1 = c1.getPath() != null ? c1.getPath() : "/";
String p2 = c2.getPath() != null ? c2.getPath() : "/";
assertEquals(p1, p2);
assertEquals(c1.getValue(), c2.getValue());
assertEquals(c1.getVersion(), c2.getVersion());
assertEquals(c1.getExpiryDate(), c2.getExpiryDate());
assertTrue(Arrays.equals(c1.getPorts(), c2.getPorts()));
}
}