/**
* Copyright 2013 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.netflix.aegisthus.input;
import java.io.BufferedInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.TypeParser;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapreduce.InputSplit;
import com.google.common.collect.Maps;
import com.netflix.aegisthus.io.sstable.compression.CompressionInputStream;
import com.netflix.aegisthus.io.sstable.compression.CompressionMetadata;
@SuppressWarnings("rawtypes")
public class AegSplit extends InputSplit implements Writable {
private static final Log LOG = LogFactory.getLog(AegSplit.class);
public enum Type {
commitlog, json, sstable
}
protected Map<String, AbstractType> convertors = null;
protected boolean compressed;
protected Path compressedPath;
protected long end;
protected String[] hosts;
protected Path path;
protected long start;
protected Type type;
protected CompressionMetadata compressionMetadata = null;
public AegSplit() {
}
public AegSplit(Path path,
long start,
long length,
String[] hosts,
Map<String, AbstractType> convertors,
boolean compressed,
Path compressedPath) {
this(path, start, length, hosts, convertors);
this.compressed = compressed;
this.compressedPath = compressedPath;
}
public AegSplit(Path path, long start, long length, String[] hosts, Map<String, AbstractType> convertors) {
this.type = Type.sstable;
this.path = path;
this.start = start;
this.end = length + start;
LOG.info(String.format("start: %d, end: %d", start, end));
this.hosts = hosts;
this.convertors = convertors;
}
public AegSplit(Path path, long start, long length, String[] hosts, Type type, Map<String, AbstractType> convertors) {
this.type = type;
this.path = path;
this.start = start;
this.end = length + start;
LOG.info(String.format("start: %d, end: %d", start, end));
this.hosts = hosts;
this.convertors = convertors;
}
public AegSplit(Path path, long start, long length, String[] hosts, Type type) {
this.type = type;
this.path = path;
this.start = start;
this.end = length + start;
LOG.info(String.format("start: %d, end: %d", start, end));
this.hosts = hosts;
}
public Path getCompressedPath() {
return compressedPath;
}
public Map<String, AbstractType> getConvertors() {
return convertors;
}
public long getEnd() {
return end;
}
@Override
public long getLength() {
return end - start;
}
@Override
public String[] getLocations() throws IOException, InterruptedException {
return hosts;
}
public Path getPath() {
return path;
}
public long getStart() {
return start;
}
public Type getType() {
return type;
}
public boolean isCompressed() {
return compressed;
}
public CompressionMetadata getCompressionMetadata() {
return compressionMetadata;
}
public InputStream getInput(Configuration conf) throws IOException {
FileSystem fs = path.getFileSystem(conf);
FSDataInputStream fileIn = fs.open(path);
InputStream dis = new DataInputStream(new BufferedInputStream(fileIn));
if (compressed) {
FSDataInputStream cmIn = fs.open(compressedPath);
compressionMetadata = new CompressionMetadata(new BufferedInputStream(cmIn), end - start);
dis = new CompressionInputStream(dis, compressionMetadata);
end = compressionMetadata.getDataLength();
}
return dis;
}
@Override
public void readFields(DataInput in) throws IOException {
end = in.readLong();
hosts = WritableUtils.readStringArray(in);
path = new Path(WritableUtils.readString(in));
compressed = in.readBoolean();
if (compressed) {
compressedPath = new Path(WritableUtils.readString(in));
}
start = in.readLong();
type = WritableUtils.readEnum(in, Type.class);
int size = in.readInt();
if (type == Type.sstable) {
convertors = Maps.newHashMap();
for (int i = 0; i < size; i++) {
String[] parts = WritableUtils.readStringArray(in);
try {
convertors.put(parts[0], TypeParser.parse(parts[1]));
} catch (ConfigurationException e) {
throw new IOException(e);
} catch (SyntaxException e) {
throw new IOException(e);
}
}
}
}
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(end);
WritableUtils.writeStringArray(out, hosts);
WritableUtils.writeString(out, path.toUri().toString());
out.writeBoolean(compressed);
if (compressed) {
WritableUtils.writeString(out, compressedPath.toUri().toString());
}
out.writeLong(start);
WritableUtils.writeEnum(out, type);
if (convertors != null) {
String[] parts = new String[2];
out.writeInt(convertors.size());
for (Map.Entry<String, AbstractType> e : convertors.entrySet()) {
parts[0] = e.getKey();
parts[1] = e.getValue().toString();
WritableUtils.writeStringArray(out, parts);
}
} else {
out.writeInt(0);
}
}
}