/*
* Copyright (C) 2011, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.storage.dht;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.Deflater;
import org.eclipse.jgit.generated.storage.dht.proto.GitStore;
import org.eclipse.jgit.generated.storage.dht.proto.GitStore.ChunkMeta;
import org.eclipse.jgit.generated.storage.dht.proto.GitStore.ChunkMeta.BaseChunk;
import org.eclipse.jgit.generated.storage.dht.proto.GitStore.ObjectInfo.ObjectType;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.storage.dht.spi.Database;
import org.eclipse.jgit.storage.dht.spi.WriteBuffer;
import org.eclipse.jgit.transport.PackedObjectInfo;
import org.eclipse.jgit.util.NB;
/**
* Formats one {@link PackChunk} for storage in the DHT.
* <p>
* Each formatter instance can be used only once.
*/
class ChunkFormatter {
static final int TRAILER_SIZE = 4;
private final RepositoryKey repo;
private final DhtInserterOptions options;
private final byte[] varIntBuf;
private final int maxObjects;
private Map<ChunkKey, BaseChunkInfo> baseChunks;
private List<StoredObject> objectList;
private byte[] chunkData;
private int ptr;
private int mark;
private int currentObjectType;
private BaseChunkInfo currentObjectBase;
private PackChunk.Members builder;
private GitStore.ChunkInfo.Source source;
private boolean fragment;
private int objectType;
private int objectsTotal, objectsWhole, objectsRefDelta, objectsOfsDelta;
private ChunkInfo chunkInfo;
ChunkFormatter(RepositoryKey repo, DhtInserterOptions options) {
this.repo = repo;
this.options = options;
this.varIntBuf = new byte[32];
this.chunkData = new byte[options.getChunkSize()];
this.maxObjects = options.getMaxObjectCount();
this.objectType = -1;
}
void setSource(GitStore.ChunkInfo.Source src) {
source = src;
}
void setObjectType(int type) {
objectType = type;
}
void setFragment() {
fragment = true;
}
ChunkKey getChunkKey() {
return getChunkInfo().getChunkKey();
}
ChunkInfo getChunkInfo() {
return chunkInfo;
}
ChunkMeta getChunkMeta() {
return builder.getMeta();
}
PackChunk getPackChunk() throws DhtException {
return builder.build();
}
void setChunkIndex(List<PackedObjectInfo> objs) {
builder.setChunkIndex(ChunkIndex.create(objs));
}
ChunkKey end(MessageDigest md) {
if (md == null)
md = Constants.newMessageDigest();
// Embed a small amount of randomness into the chunk content,
// and thus impact its name. This prevents malicious clients from
// being able to predict what a chunk is called, which keeps them
// from replacing an existing chunk.
//
chunkData = cloneArray(chunkData, ptr + TRAILER_SIZE);
NB.encodeInt32(chunkData, ptr, options.nextChunkSalt());
ptr += 4;
md.update(chunkData, 0, ptr);
ChunkKey key = ChunkKey.create(repo, ObjectId.fromRaw(md.digest()));
GitStore.ChunkInfo.Builder info = GitStore.ChunkInfo.newBuilder();
info.setSource(source);
info.setObjectType(GitStore.ChunkInfo.ObjectType.valueOf(objectType));
if (fragment)
info.setIsFragment(true);
info.setChunkSize(chunkData.length);
GitStore.ChunkInfo.ObjectCounts.Builder cnts = info.getObjectCountsBuilder();
cnts.setTotal(objectsTotal);
if (objectsWhole > 0)
cnts.setWhole(objectsWhole);
if (objectsRefDelta > 0)
cnts.setRefDelta(objectsRefDelta);
if (objectsOfsDelta > 0)
cnts.setOfsDelta(objectsOfsDelta);
builder = new PackChunk.Members();
builder.setChunkKey(key);
builder.setChunkData(chunkData);
if (baseChunks != null) {
List<BaseChunk> list = new ArrayList<BaseChunk>(baseChunks.size());
for (BaseChunkInfo b : baseChunks.values()) {
if (0 < b.useCount) {
BaseChunk.Builder c = BaseChunk.newBuilder();
c.setRelativeStart(b.relativeStart);
c.setChunkKey(b.key.asString());
list.add(c.build());
}
}
Collections.sort(list, new Comparator<BaseChunk>() {
public int compare(BaseChunk a, BaseChunk b) {
return Long.signum(a.getRelativeStart()
- b.getRelativeStart());
}
});
ChunkMeta.Builder b = ChunkMeta.newBuilder();
b.addAllBaseChunk(list);
ChunkMeta meta = b.build();
builder.setMeta(meta);
info.setMetaSize(meta.getSerializedSize());
}
if (objectList != null && !objectList.isEmpty()) {
byte[] index = ChunkIndex.create(objectList);
builder.setChunkIndex(index);
info.setIndexSize(index.length);
}
chunkInfo = new ChunkInfo(key, info.build());
return getChunkKey();
}
/**
* Safely put the chunk to the database.
* <p>
* This method is slow. It first puts the chunk info, waits for success,
* then puts the chunk itself, waits for success, and finally queues up the
* object index with its chunk links in the supplied buffer.
*
* @param db
* @param dbWriteBuffer
* @throws DhtException
*/
void safePut(Database db, WriteBuffer dbWriteBuffer) throws DhtException {
WriteBuffer chunkBuf = db.newWriteBuffer();
db.repository().put(repo, getChunkInfo(), chunkBuf);
chunkBuf.flush();
db.chunk().put(builder, chunkBuf);
chunkBuf.flush();
linkObjects(db, dbWriteBuffer);
}
void unsafePut(Database db, WriteBuffer dbWriteBuffer) throws DhtException {
db.repository().put(repo, getChunkInfo(), dbWriteBuffer);
db.chunk().put(builder, dbWriteBuffer);
linkObjects(db, dbWriteBuffer);
}
private void linkObjects(Database db, WriteBuffer dbWriteBuffer)
throws DhtException {
if (objectList != null && !objectList.isEmpty()) {
for (StoredObject obj : objectList) {
db.objectIndex().add(ObjectIndexKey.create(repo, obj),
obj.link(getChunkKey()), dbWriteBuffer);
}
}
}
boolean whole(Deflater def, int type, byte[] data, int off, final int size,
ObjectId objId) {
if (free() < 10 || maxObjects <= objectsTotal)
return false;
header(type, size);
objectsWhole++;
currentObjectType = type;
int endOfHeader = ptr;
def.setInput(data, off, size);
def.finish();
do {
int left = free();
if (left == 0) {
rollback();
return false;
}
int n = def.deflate(chunkData, ptr, left);
if (n == 0) {
rollback();
return false;
}
ptr += n;
} while (!def.finished());
if (objectList == null)
objectList = new ArrayList<StoredObject>();
final int packedSize = ptr - endOfHeader;
objectList.add(new StoredObject(objId, type, mark, packedSize, size));
if (objectType < 0)
objectType = type;
else if (objectType != type)
objectType = ChunkInfo.OBJ_MIXED;
return true;
}
boolean whole(int type, long inflatedSize) {
if (free() < 10 || maxObjects <= objectsTotal)
return false;
header(type, inflatedSize);
objectsWhole++;
currentObjectType = type;
return true;
}
boolean ofsDelta(long inflatedSize, long negativeOffset) {
final int ofsPtr = encodeVarInt(negativeOffset);
final int ofsLen = varIntBuf.length - ofsPtr;
if (free() < 10 + ofsLen || maxObjects <= objectsTotal)
return false;
header(Constants.OBJ_OFS_DELTA, inflatedSize);
objectsOfsDelta++;
currentObjectType = Constants.OBJ_OFS_DELTA;
currentObjectBase = null;
if (append(varIntBuf, ofsPtr, ofsLen))
return true;
rollback();
return false;
}
boolean refDelta(long inflatedSize, AnyObjectId baseId) {
if (free() < 30 || maxObjects <= objectsTotal)
return false;
header(Constants.OBJ_REF_DELTA, inflatedSize);
objectsRefDelta++;
currentObjectType = Constants.OBJ_REF_DELTA;
baseId.copyRawTo(chunkData, ptr);
ptr += 20;
return true;
}
void useBaseChunk(long relativeStart, ChunkKey baseChunkKey) {
if (baseChunks == null)
baseChunks = new HashMap<ChunkKey, BaseChunkInfo>();
BaseChunkInfo base = baseChunks.get(baseChunkKey);
if (base == null) {
base = new BaseChunkInfo(relativeStart, baseChunkKey);
baseChunks.put(baseChunkKey, base);
}
base.useCount++;
currentObjectBase = base;
}
void appendDeflateOutput(Deflater def) {
while (!def.finished()) {
int left = free();
if (left == 0)
return;
int n = def.deflate(chunkData, ptr, left);
if (n == 0)
return;
ptr += n;
}
}
boolean append(byte[] data, int off, int len) {
if (free() < len)
return false;
System.arraycopy(data, off, chunkData, ptr, len);
ptr += len;
return true;
}
boolean isEmpty() {
return ptr == 0;
}
int getObjectCount() {
return objectsTotal;
}
int position() {
return ptr;
}
int size() {
return ptr;
}
int free() {
return (chunkData.length - TRAILER_SIZE) - ptr;
}
byte[] getRawChunkDataArray() {
return chunkData;
}
int getCurrentObjectType() {
return currentObjectType;
}
void rollback() {
ptr = mark;
adjustObjectCount(-1, currentObjectType);
}
void adjustObjectCount(int delta, int type) {
objectsTotal += delta;
switch (type) {
case Constants.OBJ_COMMIT:
case Constants.OBJ_TREE:
case Constants.OBJ_BLOB:
case Constants.OBJ_TAG:
objectsWhole += delta;
break;
case Constants.OBJ_OFS_DELTA:
objectsOfsDelta += delta;
if (currentObjectBase != null && --currentObjectBase.useCount == 0)
baseChunks.remove(currentObjectBase.key);
currentObjectBase = null;
break;
case Constants.OBJ_REF_DELTA:
objectsRefDelta += delta;
break;
}
}
private void header(int type, long inflatedSize) {
mark = ptr;
objectsTotal++;
long nextLength = inflatedSize >>> 4;
chunkData[ptr++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (inflatedSize & 0x0F));
inflatedSize = nextLength;
while (inflatedSize > 0) {
nextLength >>>= 7;
chunkData[ptr++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (inflatedSize & 0x7F));
inflatedSize = nextLength;
}
}
private int encodeVarInt(long value) {
int n = varIntBuf.length - 1;
varIntBuf[n] = (byte) (value & 0x7F);
while ((value >>= 7) > 0)
varIntBuf[--n] = (byte) (0x80 | (--value & 0x7F));
return n;
}
private static byte[] cloneArray(byte[] src, int len) {
byte[] dst = new byte[len];
System.arraycopy(src, 0, dst, 0, len);
return dst;
}
private static class BaseChunkInfo {
final long relativeStart;
final ChunkKey key;
int useCount;
BaseChunkInfo(long relativeStart, ChunkKey key) {
this.relativeStart = relativeStart;
this.key = key;
}
}
private static class StoredObject extends PackedObjectInfo {
private final int type;
private final int packed;
private final int inflated;
StoredObject(AnyObjectId id, int type, int offset, int packed, int size) {
super(id);
setOffset(offset);
this.type = type;
this.packed = packed;
this.inflated = size;
}
ObjectInfo link(ChunkKey key) {
GitStore.ObjectInfo.Builder b = GitStore.ObjectInfo.newBuilder();
b.setObjectType(ObjectType.valueOf(type));
b.setOffset((int) getOffset());
b.setPackedSize(packed);
b.setInflatedSize(inflated);
return new ObjectInfo(key, b.build());
}
}
}