package org.dru.clay.respository.transport.http;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URI;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.dru.clay.respository.transport.FileInfo;
import org.dru.clay.respository.transport.Transport;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpStatusCodes;
import com.google.api.client.http.javanet.NetHttpTransport;
public class HttpTransport implements Transport {
private static final Logger logger = Logger.getLogger(HttpTransport.class.getName());
private static final Pattern HREF_PATTERN = Pattern.compile("href=[\\'\"]?([^\\'\" >]+)");
private final ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() {
@Override
protected SimpleDateFormat initialValue() {
return new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z");
}
};
private final NetHttpTransport netHttpTransport;
public HttpTransport() {
netHttpTransport = new NetHttpTransport();
}
@Override
public FileInfo get(URI uri) {
if (!"http".equalsIgnoreCase(uri.getScheme())) {
throw new IllegalArgumentException("Invalid scheme for " + this.getClass().getName());
}
HttpRequest request;
try {
request = netHttpTransport.createRequestFactory().buildGetRequest(new GenericUrl(uri));
HttpResponse response = request.execute();
if (response.getStatusCode() != HttpStatusCodes.STATUS_CODE_OK) {
throw new RuntimeException("Invalid request: " + uri);
}
InputStream inputStream = response.getContent();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
int count;
byte[] buffer = new byte[8192];
while ((count = inputStream.read(buffer)) > 0) {
bos.write(buffer, 0, count);
}
bos.close();
final HttpHeaders headers = response.getHeaders();
final String lastModified = headers.getLastModified();
final Long size = headers.getContentLength() == null ? 0 : headers.getContentLength();
final Date date = lastModified == null ? new Date(0) : dateFormat.get().parse(lastModified);
return new FileInfo(size, date.getTime(), bos.toByteArray());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void get(URI uri, File destination) {
innerGet(uri, destination, true);
}
@Override
public void getIfNewer(URI uri, File destination) {
innerGet(uri, destination, false);
}
protected void innerGet(URI uri, File destination, boolean overwrite) {
if (!"http".equalsIgnoreCase(uri.getScheme())) {
throw new IllegalArgumentException("Invalid scheme for " + this.getClass().getName());
}
HttpRequest request;
try {
destination.getParentFile().mkdirs();
request = netHttpTransport.createRequestFactory().buildGetRequest(new GenericUrl(uri));
HttpResponse response = request.execute();
if (response.getStatusCode() != HttpStatusCodes.STATUS_CODE_OK) {
throw new RuntimeException("Invalid request: " + uri);
}
final HttpHeaders headers = response.getHeaders();
final String lastModified = headers.getLastModified();
final Long size = headers.getContentLength() == null ? 0 : headers.getContentLength();
final Date date = lastModified == null ? new Date(0) : dateFormat.get().parse(lastModified);
if (!overwrite && destination.exists() && destination.lastModified() >= date.getTime() && destination.length() == size) {
logger.info("File on disk is same or newer : " + destination);
return;
}
InputStream inputStream = response.getContent();
ReadableByteChannel rbc = Channels.newChannel(inputStream);
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
destination.setLastModified(date.getTime());
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
}
}
@Override
public void put(File source, URI destination) {
throw new UnsupportedOperationException("put is not implemented for Http transport");
}
@Override
public List<URI> list(URI directory) {
if (!"http".equalsIgnoreCase(directory.getScheme())) {
throw new IllegalArgumentException("Invalid scheme for " + this.getClass().getName());
}
try {
final List<URI> links = new ArrayList<URI>();
final FileInfo fileInfo = get(directory);
final String content = new String(fileInfo.getContent(), "UTF-8");
int index = 0;
final Matcher matcher = HREF_PATTERN.matcher(content);
while (matcher.find(index)) {
index = matcher.end();
final String href = matcher.group(1);
if (href.contains("://") || href.startsWith("/")) {
continue;
}
URI uri = new URI(directory + href);
if (directory.getPath().equals(uri.getPath())) {
continue;
}
links.add(uri);
}
// final Document document = Jsoup.connect(directory.toString()).get();
// for (Element element : document.select("body a")) {
// final String href = element.attr("href");
// if (href.contains("://") || href.startsWith("/")) {
// continue;
// }
// URI uri = new URI(directory + href);
// if (directory.file().equals(uri.file())) {
// continue;
// }
// links.add(uri);
// }
return links;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void cleanup() {
}
}