package com.gitblit.service;
import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.gitblit.ConfigUserService;
import com.gitblit.Constants;
import com.gitblit.Constants.AccessPermission;
import com.gitblit.Constants.FederationPullStatus;
import com.gitblit.Constants.FederationStrategy;
import com.gitblit.GitBlitException.ForbiddenException;
import com.gitblit.IUserService;
import com.gitblit.Keys;
import com.gitblit.manager.IGitblit;
import com.gitblit.models.FederationModel;
import com.gitblit.models.RefModel;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.TeamModel;
import com.gitblit.models.UserModel;
import com.gitblit.utils.ArrayUtils;
import com.gitblit.utils.FederationUtils;
import com.gitblit.utils.FileUtils;
import com.gitblit.utils.JGitUtils;
import com.gitblit.utils.JGitUtils.CloneResult;
import com.gitblit.utils.StringUtils;
public abstract class FederationPullService implements Runnable {
final Logger logger = LoggerFactory.getLogger(getClass());
final IGitblit gitblit;
private final List<FederationModel> registrations;
/**
* Constructor for specifying a single federation registration. This
* constructor is used to schedule the next pull execution.
*
* @param provider
* @param registration
*/
public FederationPullService(IGitblit gitblit, FederationModel registration) {
this(gitblit, Arrays.asList(registration));
}
/**
* Constructor to specify a group of federation registrations. This is
* normally used at startup to pull and then schedule the next update based
* on each registrations frequency setting.
*
* @param provider
* @param registrations
* @param isDaemon
* if true, registrations are rescheduled in perpetuity. if
* false, the federation pull operation is executed once.
*/
public FederationPullService(IGitblit gitblit, List<FederationModel> registrations) {
this.gitblit = gitblit;
this.registrations = registrations;
}
public abstract void reschedule(FederationModel registration);
/**
* Run method for this pull service.
*/
@Override
public void run() {
for (FederationModel registration : registrations) {
FederationPullStatus was = registration.getLowestStatus();
try {
Date now = new Date(System.currentTimeMillis());
pull(registration);
sendStatusAcknowledgment(registration);
registration.lastPull = now;
FederationPullStatus is = registration.getLowestStatus();
if (is.ordinal() < was.ordinal()) {
// the status for this registration has downgraded
logger.warn("Federation pull status of {0} is now {1}", registration.name,
is.name());
if (registration.notifyOnError) {
String message = "Federation pull of " + registration.name + " @ "
+ registration.url + " is now at " + is.name();
gitblit.sendMailToAdministrators(
"Pull Status of " + registration.name + " is " + is.name(),
message);
}
}
} catch (Throwable t) {
logger.error(MessageFormat.format(
"Failed to pull from federated gitblit ({0} @ {1})", registration.name,
registration.url), t);
} finally {
reschedule(registration);
}
}
}
/**
* Mirrors a repository and, optionally, the server's users, and/or
* configuration settings from a origin Gitblit instance.
*
* @param registration
* @throws Exception
*/
private void pull(FederationModel registration) throws Exception {
Map<String, RepositoryModel> repositories = FederationUtils.getRepositories(registration,
true);
String registrationFolder = registration.folder.toLowerCase().trim();
// confirm valid characters in server alias
Character c = StringUtils.findInvalidCharacter(registrationFolder);
if (c != null) {
logger.error(MessageFormat
.format("Illegal character ''{0}'' in folder name ''{1}'' of federation registration {2}!",
c, registrationFolder, registration.name));
return;
}
File repositoriesFolder = gitblit.getRepositoriesFolder();
File registrationFolderFile = new File(repositoriesFolder, registrationFolder);
registrationFolderFile.mkdirs();
// Clone/Pull the repository
for (Map.Entry<String, RepositoryModel> entry : repositories.entrySet()) {
String cloneUrl = entry.getKey();
RepositoryModel repository = entry.getValue();
if (!repository.hasCommits) {
logger.warn(MessageFormat.format(
"Skipping federated repository {0} from {1} @ {2}. Repository is EMPTY.",
repository.name, registration.name, registration.url));
registration.updateStatus(repository, FederationPullStatus.SKIPPED);
continue;
}
// Determine local repository name
String repositoryName;
if (StringUtils.isEmpty(registrationFolder)) {
repositoryName = repository.name;
} else {
repositoryName = registrationFolder + "/" + repository.name;
}
if (registration.bare) {
// bare repository, ensure .git suffix
if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
repositoryName += DOT_GIT_EXT;
}
} else {
// normal repository, strip .git suffix
if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
repositoryName = repositoryName.substring(0,
repositoryName.indexOf(DOT_GIT_EXT));
}
}
// confirm that the origin of any pre-existing repository matches
// the clone url
String fetchHead = null;
Repository existingRepository = gitblit.getRepository(repositoryName);
if (existingRepository == null && gitblit.isCollectingGarbage(repositoryName)) {
logger.warn(MessageFormat.format("Skipping local repository {0}, busy collecting garbage", repositoryName));
continue;
}
if (existingRepository != null) {
StoredConfig config = existingRepository.getConfig();
config.load();
String origin = config.getString("remote", "origin", "url");
RevCommit commit = JGitUtils.getCommit(existingRepository,
org.eclipse.jgit.lib.Constants.FETCH_HEAD);
if (commit != null) {
fetchHead = commit.getName();
}
existingRepository.close();
if (!origin.startsWith(registration.url)) {
logger.warn(MessageFormat
.format("Skipping federated repository {0} from {1} @ {2}. Origin does not match, consider EXCLUDING.",
repository.name, registration.name, registration.url));
registration.updateStatus(repository, FederationPullStatus.SKIPPED);
continue;
}
}
// clone/pull this repository
CredentialsProvider credentials = new UsernamePasswordCredentialsProvider(
Constants.FEDERATION_USER, registration.token);
logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",
repository.name, registration.name, registration.url));
CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,
cloneUrl, registration.bare, credentials);
Repository r = gitblit.getRepository(repositoryName);
RepositoryModel rm = gitblit.getRepositoryModel(repositoryName);
repository.isFrozen = registration.mirror;
if (result.createdRepository) {
// default local settings
repository.federationStrategy = FederationStrategy.EXCLUDE;
repository.isFrozen = registration.mirror;
repository.showRemoteBranches = !registration.mirror;
logger.info(MessageFormat.format(" cloning {0}", repository.name));
registration.updateStatus(repository, FederationPullStatus.MIRRORED);
} else {
// fetch and update
boolean fetched = false;
RevCommit commit = JGitUtils.getCommit(r, org.eclipse.jgit.lib.Constants.FETCH_HEAD);
String newFetchHead = commit.getName();
fetched = fetchHead == null || !fetchHead.equals(newFetchHead);
if (registration.mirror) {
// mirror
if (fetched) {
// update local branches to match the remote tracking branches
for (RefModel ref : JGitUtils.getRemoteBranches(r, false, -1)) {
if (ref.displayName.startsWith("origin/")) {
String branch = org.eclipse.jgit.lib.Constants.R_HEADS
+ ref.displayName.substring(ref.displayName.indexOf('/') + 1);
String hash = ref.getReferencedObjectId().getName();
JGitUtils.setBranchRef(r, branch, hash);
logger.info(MessageFormat.format(" resetting {0} of {1} to {2}", branch,
repository.name, hash));
}
}
String newHead;
if (StringUtils.isEmpty(repository.HEAD)) {
newHead = newFetchHead;
} else {
newHead = repository.HEAD;
}
JGitUtils.setHEADtoRef(r, newHead);
logger.info(MessageFormat.format(" resetting HEAD of {0} to {1}",
repository.name, newHead));
registration.updateStatus(repository, FederationPullStatus.MIRRORED);
} else {
// indicate no commits pulled
registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
}
} else {
// non-mirror
if (fetched) {
// indicate commits pulled to origin/master
registration.updateStatus(repository, FederationPullStatus.PULLED);
} else {
// indicate no commits pulled
registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
}
}
// preserve local settings
repository.isFrozen = rm.isFrozen;
repository.federationStrategy = rm.federationStrategy;
// merge federation sets
Set<String> federationSets = new HashSet<String>();
if (rm.federationSets != null) {
federationSets.addAll(rm.federationSets);
}
if (repository.federationSets != null) {
federationSets.addAll(repository.federationSets);
}
repository.federationSets = new ArrayList<String>(federationSets);
// merge indexed branches
Set<String> indexedBranches = new HashSet<String>();
if (rm.indexedBranches != null) {
indexedBranches.addAll(rm.indexedBranches);
}
if (repository.indexedBranches != null) {
indexedBranches.addAll(repository.indexedBranches);
}
repository.indexedBranches = new ArrayList<String>(indexedBranches);
}
// only repositories that are actually _cloned_ from the origin
// Gitblit repository are marked as federated. If the origin
// is from somewhere else, these repositories are not considered
// "federated" repositories.
repository.isFederated = cloneUrl.startsWith(registration.url);
gitblit.updateConfiguration(r, repository);
r.close();
}
IUserService userService = null;
try {
// Pull USERS
// TeamModels are automatically pulled because they are contained
// within the UserModel. The UserService creates unknown teams
// and updates existing teams.
Collection<UserModel> users = FederationUtils.getUsers(registration);
if (users != null && users.size() > 0) {
File realmFile = new File(registrationFolderFile, registration.name + "_users.conf");
realmFile.delete();
userService = new ConfigUserService(realmFile);
for (UserModel user : users) {
userService.updateUserModel(user.username, user);
// merge the origin permissions and origin accounts into
// the user accounts of this Gitblit instance
if (registration.mergeAccounts) {
// reparent all repository permissions if the local
// repositories are stored within subfolders
if (!StringUtils.isEmpty(registrationFolder)) {
if (user.permissions != null) {
// pulling from >= 1.2 version
Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
user.permissions.clear();
for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
user.setRepositoryPermission(registrationFolder + "/" + entry.getKey(), entry.getValue());
}
} else {
// pulling from <= 1.1 version
List<String> permissions = new ArrayList<String>(user.repositories);
user.repositories.clear();
for (String permission : permissions) {
user.addRepositoryPermission(registrationFolder + "/" + permission);
}
}
}
// insert new user or update local user
UserModel localUser = gitblit.getUserModel(user.username);
if (localUser == null) {
// create new local user
gitblit.addUser(user);
} else {
// update repository permissions of local user
if (user.permissions != null) {
// pulling from >= 1.2 version
Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
localUser.setRepositoryPermission(entry.getKey(), entry.getValue());
}
} else {
// pulling from <= 1.1 version
for (String repository : user.repositories) {
localUser.addRepositoryPermission(repository);
}
}
localUser.password = user.password;
localUser.canAdmin = user.canAdmin;
gitblit.reviseUser(localUser.username, localUser);
}
for (String teamname : gitblit.getAllTeamNames()) {
TeamModel team = gitblit.getTeamModel(teamname);
if (user.isTeamMember(teamname) && !team.hasUser(user.username)) {
// new team member
team.addUser(user.username);
gitblit.updateTeamModel(teamname, team);
} else if (!user.isTeamMember(teamname) && team.hasUser(user.username)) {
// remove team member
team.removeUser(user.username);
gitblit.updateTeamModel(teamname, team);
}
// update team repositories
TeamModel remoteTeam = user.getTeam(teamname);
if (remoteTeam != null) {
if (remoteTeam.permissions != null) {
// pulling from >= 1.2
for (Map.Entry<String, AccessPermission> entry : remoteTeam.permissions.entrySet()){
team.setRepositoryPermission(entry.getKey(), entry.getValue());
}
gitblit.updateTeamModel(teamname, team);
} else if (!ArrayUtils.isEmpty(remoteTeam.repositories)) {
// pulling from <= 1.1
team.addRepositoryPermissions(remoteTeam.repositories);
gitblit.updateTeamModel(teamname, team);
}
}
}
}
}
}
} catch (ForbiddenException e) {
// ignore forbidden exceptions
} catch (IOException e) {
logger.warn(MessageFormat.format(
"Failed to retrieve USERS from federated gitblit ({0} @ {1})",
registration.name, registration.url), e);
}
try {
// Pull TEAMS
// We explicitly pull these even though they are embedded in
// UserModels because it is possible to use teams to specify
// mailing lists or push scripts without specifying users.
if (userService != null) {
Collection<TeamModel> teams = FederationUtils.getTeams(registration);
if (teams != null && teams.size() > 0) {
for (TeamModel team : teams) {
userService.updateTeamModel(team);
}
}
}
} catch (ForbiddenException e) {
// ignore forbidden exceptions
} catch (IOException e) {
logger.warn(MessageFormat.format(
"Failed to retrieve TEAMS from federated gitblit ({0} @ {1})",
registration.name, registration.url), e);
}
try {
// Pull SETTINGS
Map<String, String> settings = FederationUtils.getSettings(registration);
if (settings != null && settings.size() > 0) {
Properties properties = new Properties();
properties.putAll(settings);
FileOutputStream os = new FileOutputStream(new File(registrationFolderFile,
registration.name + "_" + Constants.PROPERTIES_FILE));
properties.store(os, null);
os.close();
}
} catch (ForbiddenException e) {
// ignore forbidden exceptions
} catch (IOException e) {
logger.warn(MessageFormat.format(
"Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
registration.name, registration.url), e);
}
try {
// Pull SCRIPTS
Map<String, String> scripts = FederationUtils.getScripts(registration);
if (scripts != null && scripts.size() > 0) {
for (Map.Entry<String, String> script : scripts.entrySet()) {
String scriptName = script.getKey();
if (scriptName.endsWith(".groovy")) {
scriptName = scriptName.substring(0, scriptName.indexOf(".groovy"));
}
File file = new File(registrationFolderFile, registration.name + "_"
+ scriptName + ".groovy");
FileUtils.writeContent(file, script.getValue());
}
}
} catch (ForbiddenException e) {
// ignore forbidden exceptions
} catch (IOException e) {
logger.warn(MessageFormat.format(
"Failed to retrieve SCRIPTS from federated gitblit ({0} @ {1})",
registration.name, registration.url), e);
}
}
/**
* Sends a status acknowledgment to the origin Gitblit instance. This
* includes the results of the federated pull.
*
* @param registration
* @throws Exception
*/
private void sendStatusAcknowledgment(FederationModel registration) throws Exception {
if (!registration.sendStatus) {
// skip status acknowledgment
return;
}
InetAddress addr = InetAddress.getLocalHost();
String federationName = gitblit.getSettings().getString(Keys.federation.name, null);
if (StringUtils.isEmpty(federationName)) {
federationName = addr.getHostName();
}
FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
}
}