package org.apache.blur.agent;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.lang.management.ManagementFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import org.apache.blur.agent.cleaners.AgentCleaners;
import org.apache.blur.agent.collectors.blur.BlurCollector;
import org.apache.blur.agent.collectors.hdfs.HdfsCollector;
import org.apache.blur.agent.collectors.zookeeper.ZookeeperCollector;
import org.apache.blur.agent.connections.JdbcConnection;
import org.apache.blur.agent.connections.blur.BlurDatabaseConnection;
import org.apache.blur.agent.connections.cleaners.CleanerDatabaseConnection;
import org.apache.blur.agent.connections.hdfs.HdfsDatabaseConnection;
import org.apache.blur.agent.connections.zookeeper.ZookeeperDatabaseConnection;
import org.apache.blur.agent.exceptions.HdfsThreadException;
import org.apache.blur.agent.notifications.Notifier;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.PropertyConfigurator;
import org.springframework.jdbc.core.JdbcTemplate;
public class Agent {
public static final long COLLECTOR_SLEEP_TIME = TimeUnit.SECONDS.toMillis(15);
public static final long CLEAN_UP_SLEEP_TIME = TimeUnit.SECONDS.toMillis(30);
private static final Log log = LogFactory.getLog(Agent.class);
private Agent(Properties props) {
// Setup database connection
JdbcTemplate jdbc = JdbcConnection.createDBConnection(props);
// Setup the notifier
Notifier.getNotifier(props, true);
List<String> activeCollectors = props.containsKey("active.collectors") ? new ArrayList<String>(Arrays.asList(props.getProperty(
"active.collectors").split("\\|"))) : new ArrayList<String>();
// Setup the collectors
setupHdfs(props, jdbc, activeCollectors);
setupBlur(props, jdbc, activeCollectors);
setupZookeeper(props, jdbc);
setupCleaners(jdbc, activeCollectors);
}
public static void main(String[] args) {
writePidFile();
Properties configProps = loadConfigParams(args);
setupLogger(configProps);
new Agent(configProps);
}
private void setupCleaners(JdbcTemplate jdbc, List<String> activeCollectors) {
new Thread(new AgentCleaners(activeCollectors, new CleanerDatabaseConnection(jdbc)), "Agent Cleaner Thread").start();
}
private void setupBlur(Properties props, JdbcTemplate jdbc, List<String> activeCollectors) {
Map<String, String> blurInstances = loadBlurInstances(props);
for (Map.Entry<String, String> blurEntry : blurInstances.entrySet()) {
final String zookeeperName = blurEntry.getKey();
final String connection = blurEntry.getValue();
new Thread(new BlurCollector(zookeeperName, connection, activeCollectors, new BlurDatabaseConnection(jdbc), jdbc),
"Blur Collector thread - " + zookeeperName).start();
}
}
private void setupHdfs(Properties props, final JdbcTemplate jdbc, List<String> activeCollectors) {
Map<String, Map<String, String>> hdfsInstances = loadHdfsInstances(props);
for (Map<String, String> instance : hdfsInstances.values()) {
final String name = instance.get("name");
final String thriftUri = instance.get("url.thrift");
final String defaultUri = instance.get("url.default");
final String user = props.getProperty("hdfs." + name + ".login.user");
try {
new Thread(new HdfsCollector(name, defaultUri, thriftUri, user, activeCollectors, new HdfsDatabaseConnection(jdbc)),
"Hdfs Collector - " + name).start();
} catch (HdfsThreadException e) {
log.error("The collector for hdfs [" + name + "] will not execute.");
continue;
}
}
}
private void setupZookeeper(Properties props, JdbcTemplate jdbc) {
if (props.containsKey("zk.instances")) {
List<String> zooKeeperInstances = new ArrayList<String>(Arrays.asList(props.getProperty("zk.instances").split("\\|")));
for (String zkInstance : zooKeeperInstances) {
String zkUrl = props.getProperty("zk." + zkInstance + ".url");
String blurConnection = props.getProperty("blur." + zkInstance + ".url");
new Thread(new ZookeeperCollector(zkUrl, zkInstance, blurConnection, new ZookeeperDatabaseConnection(jdbc)), "Zookeeper - "
+ zkInstance).start();
}
}
}
private static void setupLogger(Properties props) {
String log4jPropsFile = props.getProperty("log4j.properties", "../conf/log4j.properties");
if (new File(log4jPropsFile).exists()) {
PropertyConfigurator.configure(log4jPropsFile);
} else {
log.warn("Unable to find log4j properties file. Using default logging");
}
}
private static Properties loadConfigParams(String[] args) {
String configFileName;
if (args.length == 0) {
configFileName = "../conf/blur-agent.config";
} else {
configFileName = args[0];
}
File configFile = new File(configFileName);
if (!configFile.exists() || !configFile.isFile()) {
log.fatal("Unable to find config file at " + configFile.getAbsolutePath());
System.exit(1);
}
Properties configProps = new Properties();
try {
configProps.load(new FileInputStream(configFile));
} catch (Exception e) {
log.fatal("Config File is not a valid properties file: " + e.getMessage());
System.exit(1);
}
return configProps;
}
private static void writePidFile() {
try {
File pidFile = new File("../agent.pid");
PrintWriter pidOut = new PrintWriter(pidFile);
log.info("Wrote pid file to: " + pidFile.getAbsolutePath());
String nameOfRunningVM = ManagementFactory.getRuntimeMXBean().getName();
int p = nameOfRunningVM.indexOf('@');
String pid = nameOfRunningVM.substring(0, p);
pidOut.write(pid);
pidOut.write("\n");
pidOut.close();
} catch (FileNotFoundException e) {
log.fatal("Unable to find pid file. " + e.getMessage());
System.exit(1);
}
}
private Map<String, String> loadBlurInstances(Properties props) {
Map<String, String> instances = new HashMap<String, String>();
if (props.containsKey("blur.instances")) {
String[] blurNames = props.getProperty("blur.instances").split("\\|");
for (String blur : blurNames) {
instances.put(blur, props.getProperty("blur." + blur + ".url"));
}
}
return instances;
}
private Map<String, Map<String, String>> loadHdfsInstances(Properties props) {
Map<String, Map<String, String>> instances = new HashMap<String, Map<String, String>>();
if (props.containsKey("hdfs.instances")) {
String[] hdfsNames = props.getProperty("hdfs.instances").split("\\|");
for (String hdfs : hdfsNames) {
Map<String, String> instanceInfo = new HashMap<String, String>();
instanceInfo.put("url.thrift", props.getProperty("hdfs." + hdfs + ".thrift.url"));
instanceInfo.put("url.default", props.getProperty("hdfs." + hdfs + ".url"));
instanceInfo.put("name", hdfs);
instances.put(hdfs, instanceInfo);
}
}
return instances;
}
}