/* Copyright (c) 2013-2014 Boundless and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Distribution License v1.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/org/documents/edl-v10.html
*
* Contributors:
* Johnathan Garrett (LMN Solutions) - initial implementation
*/
package org.locationtech.geogig.remote;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.locationtech.geogig.api.ObjectId;
import org.locationtech.geogig.api.Ref;
import org.locationtech.geogig.api.RepositoryFilter.FilterDescription;
import org.locationtech.geogig.api.RevCommit;
import org.locationtech.geogig.api.RevObject;
import org.locationtech.geogig.api.RevObject.TYPE;
import org.locationtech.geogig.api.SymRef;
import org.locationtech.geogig.api.plumbing.CheckSparsePath;
import org.locationtech.geogig.api.plumbing.FindCommonAncestor;
import org.locationtech.geogig.api.plumbing.RevObjectParse;
import org.locationtech.geogig.api.plumbing.diff.DiffEntry;
import org.locationtech.geogig.api.porcelain.DiffOp;
import org.locationtech.geogig.repository.Repository;
import org.locationtech.geogig.storage.ObjectSerializingFactory;
import org.locationtech.geogig.storage.ObjectWriter;
import org.locationtech.geogig.storage.datastream.DataStreamSerializationFactoryV1;
import com.google.common.base.Optional;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
/**
* Provides a means of communicating between a local sparse clone and a remote http full repository.
*
* @see AbstractMappedRemoteRepo
*/
class HttpMappedRemoteRepo extends AbstractMappedRemoteRepo {
private URL repositoryURL;
/**
* Constructs a new {@code HttpMappedRemoteRepo}.
*
* @param repositoryURL the URL of the repository
* @param localRepository the local sparse repository
*/
public HttpMappedRemoteRepo(URL repositoryURL, Repository localRepository) {
super(localRepository);
String url = repositoryURL.toString();
if (url.endsWith("/")) {
url = url.substring(0, url.lastIndexOf('/'));
}
try {
this.repositoryURL = new URL(url);
} catch (MalformedURLException e) {
this.repositoryURL = repositoryURL;
}
}
/**
* Currently does nothing for HTTP Remote.
*
* @throws IOException
*/
@Override
public void open() throws IOException {
}
/**
* Currently does nothing for HTTP Remote.
*
* @throws IOException
*/
@Override
public void close() throws IOException {
}
/**
* List the mapped versions of the remote's {@link Ref refs}. For example, if the remote ref
* points to commit A, the returned ref will point to the commit that A is mapped to.
*
* @param getHeads whether to return refs in the {@code refs/heads} namespace
* @param getTags whether to return refs in the {@code refs/tags} namespace
* @return an immutable set of refs from the remote
*/
@Override
public ImmutableSet<Ref> listRefs(boolean getHeads, boolean getTags) {
HttpURLConnection connection = null;
ImmutableSet.Builder<Ref> builder = new ImmutableSet.Builder<Ref>();
try {
String expanded = repositoryURL.toString() + "/repo/manifest";
connection = (HttpURLConnection) new URL(expanded).openConnection();
connection.setRequestMethod("GET");
connection.setUseCaches(false);
connection.setDoOutput(true);
// Get Response
InputStream is = connection.getInputStream();
BufferedReader rd = new BufferedReader(new InputStreamReader(is));
String line;
try {
while ((line = rd.readLine()) != null) {
if ((getHeads && line.startsWith("refs/heads"))
|| (getTags && line.startsWith("refs/tags"))) {
Ref remoteRef = HttpUtils.parseRef(line);
Ref newRef = remoteRef;
if (!(newRef instanceof SymRef)
&& localRepository.graphDatabase().exists(remoteRef.getObjectId())) {
ObjectId mappedCommit = localRepository.graphDatabase().getMapping(
remoteRef.getObjectId());
if (mappedCommit != null) {
newRef = new Ref(remoteRef.getName(), mappedCommit);
}
}
builder.add(newRef);
}
}
} finally {
rd.close();
}
} catch (Exception e) {
throw Throwables.propagate(e);
} finally {
HttpUtils.consumeErrStreamAndCloseConnection(connection);
}
return builder.build();
}
/**
* @return the remote's HEAD {@link Ref}.
*/
@Override
public Ref headRef() {
HttpURLConnection connection = null;
Ref headRef = null;
try {
String expanded = repositoryURL.toString() + "/repo/manifest";
connection = (HttpURLConnection) new URL(expanded).openConnection();
connection.setRequestMethod("GET");
connection.setUseCaches(false);
connection.setDoOutput(true);
// Get Response
InputStream is = connection.getInputStream();
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is));
String line;
while ((line = rd.readLine()) != null) {
if (line.startsWith("HEAD")) {
headRef = HttpUtils.parseRef(line);
}
}
rd.close();
} finally {
is.close();
}
} catch (Exception e) {
Throwables.propagate(e);
} finally {
HttpUtils.consumeErrStreamAndCloseConnection(connection);
}
return headRef;
}
/**
* Delete a {@link Ref} from the remote repository.
*
* @param refspec the ref to delete
*/
@Override
public Optional<Ref> deleteRef(String refspec) {
return updateRemoteRef(refspec, null, true);
}
/**
* @return the {@link RepositoryWrapper} for this remote
*/
@Override
protected RepositoryWrapper getRemoteWrapper() {
return new HttpRepositoryWrapper(repositoryURL);
}
/**
* Gets all of the changes from the target commit that should be applied to the sparse clone.
*
* @param commit the commit to get changes from
* @return an iterator for changes that match the repository filter
*/
@Override
protected FilteredDiffIterator getFilteredChanges(RevCommit commit) {
// Get affected features
ImmutableList<ObjectId> affectedFeatures = HttpUtils.getAffectedFeatures(repositoryURL,
commit.getId());
// Create a list of features I have
List<ObjectId> tracked = new LinkedList<ObjectId>();
for (ObjectId id : affectedFeatures) {
if (localRepository.blobExists(id)) {
tracked.add(id);
}
}
// Get changes from commit, pass filter and my list of features
final JsonObject message = createFetchMessage(commit.getId(), tracked);
final URL resourceURL;
try {
resourceURL = new URL(repositoryURL.toString() + "/repo/filteredchanges");
} catch (MalformedURLException e) {
throw Throwables.propagate(e);
}
final Gson gson = new Gson();
final HttpURLConnection connection;
final OutputStream out;
final Writer writer;
try {
connection = (HttpURLConnection) resourceURL.openConnection();
connection.setDoOutput(true);
connection.setDoInput(true);
out = connection.getOutputStream();
writer = new OutputStreamWriter(out);
gson.toJson(message, writer);
writer.flush();
} catch (IOException e) {
throw Throwables.propagate(e);
}
final InputStream in;
try {
in = connection.getInputStream();
} catch (IOException e) {
throw Throwables.propagate(e);
}
BinaryPackedChanges unpacker = new BinaryPackedChanges(localRepository);
return new HttpFilteredDiffIterator(in, unpacker);
}
private JsonObject createFetchMessage(ObjectId commitId, List<ObjectId> tracked) {
JsonObject message = new JsonObject();
JsonArray trackedArray = new JsonArray();
for (ObjectId id : tracked) {
trackedArray.add(new JsonPrimitive(id.toString()));
}
message.add("commitId", new JsonPrimitive(commitId.toString()));
message.add("tracked", trackedArray);
JsonArray filterArray = new JsonArray();
ImmutableList<FilterDescription> repoFilters = filter.getFilterDescriptions();
for (FilterDescription description : repoFilters) {
JsonObject typeFilter = new JsonObject();
typeFilter.add("featurepath", new JsonPrimitive(description.getFeaturePath()));
typeFilter.add("type", new JsonPrimitive(description.getFilterType()));
typeFilter.add("filter", new JsonPrimitive(description.getFilter()));
filterArray.add(typeFilter);
}
message.add("filter", filterArray);
return message;
}
/**
* Gets the remote ref that matches the provided ref spec.
*
* @param refspec the refspec to parse
* @return the matching {@link Ref} or {@link Optional#absent()} if the ref could not be found
*/
@Override
protected Optional<Ref> getRemoteRef(String refspec) {
return HttpUtils.getRemoteRef(repositoryURL, refspec);
}
/**
* Perform pre-push actions.
*/
@Override
protected void beginPush() {
HttpUtils.beginPush(repositoryURL);
}
/**
* Perform post-push actions, this includes verification that the remote wasn't changed while we
* were pushing.
*
* @param refspec the refspec that we are pushing to
* @param newCommitId the new commit id
* @param originalRefValue the original value of the ref before pushing
*/
@Override
protected void endPush(String refspec, ObjectId newCommitId, String originalRefValue) {
HttpUtils.endPush(repositoryURL, refspec, newCommitId, originalRefValue);
}
/**
* Pushes a sparse commit to a remote repository and updates all mappings.
*
* @param commitId the commit to push
*/
@Override
protected void pushSparseCommit(ObjectId commitId) {
Repository from = localRepository;
Optional<RevObject> object = from.command(RevObjectParse.class).setObjectId(commitId)
.call();
if (object.isPresent() && object.get().getType().equals(TYPE.COMMIT)) {
RevCommit commit = (RevCommit) object.get();
ObjectId parent = ObjectId.NULL;
List<ObjectId> newParents = new LinkedList<ObjectId>();
for (int i = 0; i < commit.getParentIds().size(); i++) {
ObjectId parentId = commit.getParentIds().get(i);
if (i != 0) {
Optional<ObjectId> commonAncestor = from.command(FindCommonAncestor.class)
.setLeftId(commit.getParentIds().get(0)).setRightId(parentId).call();
if (commonAncestor.isPresent()) {
if (from.command(CheckSparsePath.class).setStart(parentId)
.setEnd(commonAncestor.get()).call()) {
// This should be the base commit to preserve changes that were filtered
// out.
newParents.add(0, from.graphDatabase().getMapping(parentId));
continue;
}
}
}
newParents.add(from.graphDatabase().getMapping(parentId));
}
if (newParents.size() > 0) {
parent = from.graphDatabase().getMapping(newParents.get(0));
}
Iterator<DiffEntry> diffIter = from.command(DiffOp.class).setNewVersion(commitId)
.setOldVersion(parent).setReportTrees(true).call();
// connect and send packed changes
final URL resourceURL;
try {
resourceURL = new URL(repositoryURL.toString() + "/repo/applychanges");
} catch (MalformedURLException e) {
throw Throwables.propagate(e);
}
final HttpURLConnection connection;
final OutputStream out;
try {
connection = (HttpURLConnection) resourceURL.openConnection();
connection.setDoOutput(true);
connection.setDoInput(true);
out = connection.getOutputStream();
// pack the commit object
final ObjectSerializingFactory factory = DataStreamSerializationFactoryV1.INSTANCE;
final ObjectWriter<RevCommit> commitWriter = factory
.createObjectWriter(TYPE.COMMIT);
commitWriter.write(commit, out);
// write the new parents
out.write(newParents.size());
for (ObjectId parentId : newParents) {
out.write(parentId.getRawValue());
}
// pack the changes
BinaryPackedChanges changes = new BinaryPackedChanges(from);
changes.write(out, diffIter);
} catch (IOException e) {
throw Throwables.propagate(e);
}
final InputStream in;
try {
in = connection.getInputStream();
BufferedReader rd = new BufferedReader(new InputStreamReader(in));
String line = rd.readLine();
if (line != null) {
ObjectId remoteCommitId = ObjectId.valueOf(line);
from.graphDatabase().map(commit.getId(), remoteCommitId);
from.graphDatabase().map(remoteCommitId, commit.getId());
}
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
}
/**
* Retrieves an object with the specified id from the remote.
*
* @param objectId the object to get
* @return the fetched object
*/
@Override
protected Optional<RevObject> getObject(ObjectId objectId) {
return HttpUtils.getNetworkObject(repositoryURL, null, objectId);
}
/**
* Updates the remote ref that matches the given refspec.
*
* @param refspec the ref to update
* @param commitId the new value of the ref
* @param delete if true, the remote ref will be deleted
* @return the updated ref
*/
@Override
protected Optional<Ref> updateRemoteRef(String refspec, ObjectId commitId, boolean delete) {
return HttpUtils.updateRemoteRef(repositoryURL, refspec, commitId, delete);
}
/**
* Gets the depth of the remote repository.
*
* @return the depth of the repository, or {@link Optional#absent()} if the repository is not
* shallow
*/
@Override
public Optional<Integer> getDepth() {
return HttpUtils.getDepth(repositoryURL, null);
}
}