/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.jackrabbit.oak.plugins.blob.datastore;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.SequenceInputStream;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import javax.annotation.Nullable;
import javax.jcr.RepositoryException;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.Weigher;
import com.google.common.collect.Iterators;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closeables;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.core.data.MultiDataStoreAware;
import org.apache.jackrabbit.oak.cache.CacheLIRS;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterators.filter;
import static com.google.common.collect.Iterators.transform;
/**
* BlobStore wrapper for DataStore. Wraps Jackrabbit 2 DataStore and expose them as BlobStores
* It also handles inlining binaries if there size is smaller than
* {@link org.apache.jackrabbit.core.data.DataStore#getMinRecordLength()}
*/
public class DataStoreBlobStore implements DataStore, BlobStore, GarbageCollectableBlobStore {
private final Logger log = LoggerFactory.getLogger(getClass());
private final DataStore delegate;
/**
* If set to true then the blob length information would be encoded as part of blobId
* and thus no extra call would be made to DataStore to determine the length
*
* <b>Implementation Note</b>If enabled the length would be encoded in blobid by appending it at the end.
* This would be done for the methods which are part of BlobStore and GarbageCollectableBlobStore interface
*
* DataIdentifiers which are part of DataStore would not be affected by this as DataStore interface
* is not used in Oak and all access is via BlobStore interface
*/
private final boolean encodeLengthInId;
protected final LoadingCache<String, byte[]> cache;
public static final int DEFAULT_CACHE_SIZE = 16;
/**
* Max size of binary whose content would be cached. We keep it greater than
* Lucene blob size OakDirectory#BLOB_SIZE such that Lucene index blobs are cached
*/
private int maxCachedBinarySize = 17 * 1024;
public DataStoreBlobStore(DataStore delegate) {
this(delegate, true, DEFAULT_CACHE_SIZE);
}
public DataStoreBlobStore(DataStore delegate, boolean encodeLengthInId) {
this(delegate, encodeLengthInId, DEFAULT_CACHE_SIZE);
}
public DataStoreBlobStore(DataStore delegate, boolean encodeLengthInId, int cacheSizeInMB) {
this.delegate = delegate;
this.encodeLengthInId = encodeLengthInId;
this.cache = CacheLIRS.newBuilder()
.maximumWeight(cacheSizeInMB * FileUtils.ONE_MB)
.weigher(new Weigher<String, byte[]>() {
@Override
public int weigh(String key, byte[] value) {
return value.length;
}
})
.build();
}
//~----------------------------------< DataStore >
@Override
public DataRecord getRecordIfStored(DataIdentifier identifier) throws DataStoreException {
if(isInMemoryRecord(identifier)){
return getDataRecord(identifier.toString());
}
return delegate.getRecordIfStored(identifier);
}
@Override
public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
if(isInMemoryRecord(identifier)){
return getDataRecord(identifier.toString());
}
return delegate.getRecord(identifier);
}
@Override
public DataRecord getRecordFromReference(String reference) throws DataStoreException {
return delegate.getRecordFromReference(reference);
}
@Override
public DataRecord addRecord(InputStream stream) throws DataStoreException {
try {
return writeStream(stream);
} catch (IOException e) {
throw new DataStoreException(e);
}
}
@Override
public void updateModifiedDateOnAccess(long before) {
delegate.updateModifiedDateOnAccess(before);
}
@Override
public int deleteAllOlderThan(long min) throws DataStoreException {
return delegate.deleteAllOlderThan(min);
}
@Override
public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException {
return delegate.getAllIdentifiers();
}
@Override
public void init(String homeDir) throws RepositoryException {
throw new UnsupportedOperationException("DataStore cannot be initialized again");
}
@Override
public int getMinRecordLength() {
return delegate.getMinRecordLength();
}
@Override
public void close() throws DataStoreException {
delegate.close();
cache.invalidateAll();
}
//~-------------------------------------------< BlobStore >
@Override
public String writeBlob(InputStream stream) throws IOException {
boolean threw = true;
try {
checkNotNull(stream);
DataRecord dr = writeStream(stream);
String id = getBlobId(dr);
threw = false;
return id;
} catch (DataStoreException e) {
throw new IOException(e);
} finally {
//DataStore does not closes the stream internally
//So close the stream explicitly
Closeables.close(stream, threw);
}
}
@Override
public int readBlob(String encodedBlobId, long pos, byte[] buff, int off, int length) throws IOException {
//This is inefficient as repeated calls for same blobId would involve opening new Stream
//instead clients should directly access the stream from DataRecord by special casing for
//BlobStore which implements DataStore
InputStream stream = getInputStream(encodedBlobId);
boolean threw = true;
try {
ByteStreams.skipFully(stream, pos);
int readCount = stream.read(buff, off, length);
threw = false;
return readCount;
} finally {
Closeables.close(stream, threw);
}
}
@Override
public long getBlobLength(String encodedBlobId) throws IOException {
try {
checkNotNull(encodedBlobId, "BlobId must be specified");
BlobId id = BlobId.of(encodedBlobId);
if(encodeLengthInId && id.hasLengthInfo()){
return id.length;
}
return getDataRecord(id.blobId).getLength();
} catch (DataStoreException e) {
throw new IOException(e);
}
}
@Override
public String getBlobId(String reference) {
checkNotNull(reference);
DataRecord record;
try {
record = delegate.getRecordFromReference(reference);
if (record != null) {
return getBlobId(record);
}
} catch (DataStoreException e) {
log.warn("Unable to access the blobId for [{}]", reference, e);
}
return null;
}
@Override
public String getReference(String encodedBlobId) {
checkNotNull(encodedBlobId);
String blobId = extractBlobId(encodedBlobId);
//Reference are not created for in memory record
if(InMemoryDataRecord.isInstance(blobId)){
return null;
}
DataRecord record;
try {
record = delegate.getRecord(new DataIdentifier(blobId));
if(record != null){
return record.getReference();
}else{
log.debug("No blob found for id [{}]", blobId);
}
} catch (DataStoreException e) {
log.warn("Unable to access the blobId for [{}]", blobId, e);
}
return null;
}
@Override
public InputStream getInputStream(final String encodedBlobId) throws IOException {
final BlobId blobId = BlobId.of(encodedBlobId);
if (encodeLengthInId
&& blobId.hasLengthInfo()
&& blobId.length <= maxCachedBinarySize) {
try {
byte[] content = cache.get(blobId.blobId, new Callable<byte[]>() {
@Override
public byte[] call() throws Exception {
boolean threw = true;
InputStream stream = getStream(blobId.blobId);
try {
byte[] result = IOUtils.toByteArray(stream);
threw = false;
return result;
} finally {
Closeables.close(stream, threw);
}
}
});
return new ByteArrayInputStream(content);
} catch (ExecutionException e) {
log.warn("Error occurred while loading bytes from steam while fetching for id {}", encodedBlobId);
}
}
return getStream(blobId.blobId);
}
//~-------------------------------------------< GarbageCollectableBlobStore >
@Override
public void setBlockSize(int x) {
}
@Override
public String writeBlob(String tempFileName) throws IOException {
File file = new File(tempFileName);
InputStream in = null;
try {
in = new FileInputStream(file);
return writeBlob(in);
} finally {
org.apache.commons.io.IOUtils.closeQuietly(in);
FileUtils.forceDelete(file);
}
}
@Override
public int sweep() throws IOException {
return 0;
}
@Override
public void startMark() throws IOException {
}
@Override
public void clearInUse() {
delegate.clearInUse();
}
@Override
public void clearCache() {
}
@Override
public long getBlockSizeMin() {
return 0;
}
@Override
public Iterator<String> getAllChunkIds(final long maxLastModifiedTime) throws Exception {
return transform(filter(transform(delegate.getAllIdentifiers(), new Function<DataIdentifier, DataRecord>() {
@Nullable
@Override
public DataRecord apply(@Nullable DataIdentifier input) {
try {
return delegate.getRecord(input);
} catch (DataStoreException e) {
log.warn("Error occurred while fetching DataRecord for identifier {}", input, e);
}
return null;
}
}), new Predicate<DataRecord>() {
@Override
public boolean apply(@Nullable DataRecord input) {
if (input != null && (maxLastModifiedTime <= 0
|| input.getLastModified() < maxLastModifiedTime)) {
return true;
}
return false;
}
}),new Function<DataRecord, String>() {
@Override
public String apply(DataRecord input) {
if(encodeLengthInId) {
return BlobId.of(input).encodedValue();
}
return input.getIdentifier().toString();
}
});
}
@Override
public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
if (delegate instanceof MultiDataStoreAware) {
for (String chunkId : chunkIds) {
String blobId = extractBlobId(chunkId);
DataIdentifier identifier = new DataIdentifier(blobId);
DataRecord dataRecord = delegate.getRecord(identifier);
boolean success = (maxLastModifiedTime <= 0)
|| dataRecord.getLastModified() <= maxLastModifiedTime;
if (success) {
((MultiDataStoreAware) delegate).deleteRecord(identifier);
}
}
}
return true;
}
@Override
public Iterator<String> resolveChunks(String blobId) throws IOException {
return Iterators.singletonIterator(blobId);
}
//~---------------------------------------------< Object >
@Override
public String toString() {
return String.format("DataStore backed BlobStore [%s]", delegate.getClass().getName());
}
//~---------------------------------------------< Properties >
public DataStore getDataStore() {
return delegate;
}
public void setMaxCachedBinarySize(int maxCachedBinarySize) {
this.maxCachedBinarySize = maxCachedBinarySize;
}
//~---------------------------------------------< Internal >
private InputStream getStream(String blobId) throws IOException {
try {
return getDataRecord(blobId).getStream();
} catch (DataStoreException e) {
throw new IOException(e);
}
}
private DataRecord getDataRecord(String blobId) throws DataStoreException {
DataRecord id;
if(InMemoryDataRecord.isInstance(blobId)){
id = InMemoryDataRecord.getInstance(blobId);
}else{
id = delegate.getRecord(new DataIdentifier(blobId));
}
checkNotNull(id, "No DataRecord found for blobId [%s]", blobId);
return id;
}
private boolean isInMemoryRecord(DataIdentifier identifier){
return InMemoryDataRecord.isInstance(identifier.toString());
}
/**
* Create a BLOB value from in input stream. Small objects will create an in-memory object,
* while large objects are stored in the data store
*
* @param in the input stream
* @return the value
*/
private DataRecord writeStream(InputStream in) throws IOException, DataStoreException {
int maxMemorySize = Math.max(0, delegate.getMinRecordLength() + 1);
byte[] buffer = new byte[maxMemorySize];
int pos = 0, len = maxMemorySize;
while (pos < maxMemorySize) {
int l = in.read(buffer, pos, len);
if (l < 0) {
break;
}
pos += l;
len -= l;
}
DataRecord record;
if (pos < maxMemorySize) {
// shrink the buffer
byte[] data = new byte[pos];
System.arraycopy(buffer, 0, data, 0, pos);
record = InMemoryDataRecord.getInstance(data);
} else {
// a few bytes are already read, need to re-build the input stream
in = new SequenceInputStream(new ByteArrayInputStream(buffer, 0, pos), in);
record = delegate.addRecord(in);
}
return record;
}
private String getBlobId(DataRecord dr){
if(encodeLengthInId){
return BlobId.of(dr).encodedValue();
}
return dr.getIdentifier().toString();
}
private String extractBlobId(String encodedBlobId){
if(encodeLengthInId){
return BlobId.of(encodedBlobId).blobId;
}
return encodedBlobId;
}
static class BlobId {
static final String SEP = "#";
final String blobId;
final long length;
BlobId(String blobId, long length) {
this.blobId = blobId;
this.length = length;
}
BlobId(DataRecord dr) {
this.blobId = dr.getIdentifier().toString();
long len;
try {
len = dr.getLength();
} catch (DataStoreException e) {
//Cannot determine length
len = -1;
}
this.length = len;
}
BlobId(String encodedBlobId) {
int indexOfSep = encodedBlobId.lastIndexOf(SEP);
if(indexOfSep != -1){
this.blobId = encodedBlobId.substring(0, indexOfSep);
this.length = Long.valueOf(encodedBlobId.substring(indexOfSep+SEP.length()));
}else{
this.blobId = encodedBlobId;
this.length = -1;
}
}
String encodedValue(){
if(hasLengthInfo()){
return blobId + SEP + String.valueOf(length);
} else{
return blobId;
}
}
boolean hasLengthInfo(){
return length != -1;
}
static boolean isEncoded(String encodedBlobId){
return encodedBlobId.contains(SEP);
}
static BlobId of(String encodedValue){
return new BlobId(encodedValue);
}
static BlobId of(DataRecord dr){
return new BlobId(dr);
}
}
}