Package org.apache.hadoop.conf

Examples of org.apache.hadoop.conf.Configuration


   * run in their own thread rather than within the context of the constructor.
   * @throws InterruptedException
   */
  public HMaster(final Configuration conf)
  throws IOException, KeeperException, InterruptedException {
    this.conf = new Configuration(conf);
    // Disable the block cache on the master
    this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
    // Set how many times to retry talking to another server over HConnection.
    HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
    // Server to handle client requests.
View Full Code Here


   */
  public static void main(String[] args) throws Exception {
    Log LOG = LogFactory.getLog("RESTServer");

    VersionInfo.logVersion();
    Configuration conf = HBaseConfiguration.create();
    // login the server principal (if using secure Hadoop)
    if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
      String machineName = Strings.domainNamePointerToHostName(
        DNS.getDefaultHost(conf.get("hbase.rest.dns.interface", "default"),
          conf.get("hbase.rest.dns.nameserver", "default")));
      User.login(conf, "hbase.rest.keytab.file", "hbase.rest.kerberos.principal",
        machineName);
    }

    RESTServlet servlet = RESTServlet.getInstance(conf);

    Options options = new Options();
    options.addOption("p", "port", true, "Port to bind to [default: 8080]");
    options.addOption("ro", "readonly", false, "Respond only to GET HTTP " +
      "method requests [default: false]");
    options.addOption(null, "infoport", true, "Port for web UI");

    CommandLine commandLine = null;
    try {
      commandLine = new PosixParser().parse(options, args);
    } catch (ParseException e) {
      LOG.error("Could not parse: ", e);
      printUsageAndExit(options, -1);
    }

    // check for user-defined port setting, if so override the conf
    if (commandLine != null && commandLine.hasOption("port")) {
      String val = commandLine.getOptionValue("port");
      servlet.getConfiguration()
          .setInt("hbase.rest.port", Integer.valueOf(val));
      LOG.debug("port set to " + val);
    }

    // check if server should only process GET requests, if so override the conf
    if (commandLine != null && commandLine.hasOption("readonly")) {
      servlet.getConfiguration().setBoolean("hbase.rest.readonly", true);
      LOG.debug("readonly set to true");
    }

    // check for user-defined info server port setting, if so override the conf
    if (commandLine != null && commandLine.hasOption("infoport")) {
      String val = commandLine.getOptionValue("infoport");
      servlet.getConfiguration()
          .setInt("hbase.rest.info.port", Integer.valueOf(val));
      LOG.debug("Web UI port set to " + val);
    }

    @SuppressWarnings("unchecked")
    List<String> remainingArgs = commandLine != null ?
        commandLine.getArgList() : new ArrayList<String>();
    if (remainingArgs.size() != 1) {
      printUsageAndExit(options, 1);
    }

    String command = remainingArgs.get(0);
    if ("start".equals(command)) {
      // continue and start container
    } else if ("stop".equals(command)) {
      System.exit(1);
    } else {
      printUsageAndExit(options, 1);
    }

    // set up the Jersey servlet container for Jetty
    ServletHolder sh = new ServletHolder(ServletContainer.class);
    sh.setInitParameter(
      "com.sun.jersey.config.property.resourceConfigClass",
      ResourceConfig.class.getCanonicalName());
    sh.setInitParameter("com.sun.jersey.config.property.packages",
      "jetty");

    // set up Jetty and run the embedded server

    Server server = new Server();

    Connector connector = new SelectChannelConnector();
    connector.setPort(servlet.getConfiguration().getInt("hbase.rest.port", 8080));
    connector.setHost(servlet.getConfiguration().get("hbase.rest.host", "0.0.0.0"));

    server.addConnector(connector);

    // Set the default max thread number to 100 to limit
    // the number of concurrent requests so that REST server doesn't OOM easily.
    // Jetty set the default max thread number to 250, if we don't set it.
    //
    // Our default min thread number 2 is the same as that used by Jetty.
    int maxThreads = servlet.getConfiguration().getInt("hbase.rest.threads.max", 100);
    int minThreads = servlet.getConfiguration().getInt("hbase.rest.threads.min", 2);
    QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads);
    threadPool.setMinThreads(minThreads);
    server.setThreadPool(threadPool);

    server.setSendServerVersion(false);
    server.setSendDateHeader(false);
    server.setStopAtShutdown(true);

    // set up context
    Context context = new Context(server, "/", Context.SESSIONS);
    context.addServlet(sh, "/*");
    context.addFilter(GzipFilter.class, "/*", 0);

    // Put up info server.
    int port = conf.getInt("hbase.rest.info.port", 8085);
    if (port >= 0) {
      conf.setLong("startcode", System.currentTimeMillis());
      String a = conf.get("hbase.rest.info.bindAddress", "0.0.0.0");
      InfoServer infoServer = new InfoServer("rest", a, port, false, conf);
      infoServer.setAttribute("hbase.conf", conf);
      infoServer.start();
    }

View Full Code Here

   * @throws MasterNotRunningException if the master is not running
   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
   */
  public static void checkHBaseAvailable(Configuration conf)
  throws MasterNotRunningException, ZooKeeperConnectionException {
    Configuration copyOfConf = HBaseConfiguration.create(conf);
    copyOfConf.setInt("hbase.client.retries.number", 1);
    HBaseAdmin admin = new HBaseAdmin(copyOfConf);
    try {
      admin.close();
    } catch (IOException ioe) {
      admin.LOG.info("Failed to close connection", ioe);
View Full Code Here

   *         use HRegionInfo.getTableNameAsString() in place of
   *         HRegionInfo.getTableDesc().getNameAsString()
   */
   @Deprecated
  public HTableDescriptor getTableDesc() {
    Configuration c = HBaseConfiguration.create();
    c.set("fs.defaultFS", c.get(HConstants.HBASE_DIR));
    c.set("fs.default.name", c.get(HConstants.HBASE_DIR));
    FileSystem fs;
    try {
      fs = FileSystem.get(c);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    FSTableDescriptors fstd =
      new FSTableDescriptors(fs, new Path(c.get(HConstants.HBASE_DIR)));
    try {
      return fstd.get(this.tableName);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
View Full Code Here

      throws IOException {
    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

    final List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
    final String table = snapshotDesc.getTable();
    final Configuration conf = getConf();

    // Get snapshot files
    SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
      new SnapshotReferenceUtil.FileVisitor() {
        public void storeFile (final String region, final String family, final String hfile)
View Full Code Here

   */
  private boolean runCopyJob(final Path inputRoot, final Path outputRoot,
      final List<Pair<Path, Long>> snapshotFiles, final boolean verifyChecksum,
      final String filesUser, final String filesGroup, final int filesMode,
      final int mappers) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = getConf();
    if (filesGroup != null) conf.set(CONF_FILES_GROUP, filesGroup);
    if (filesUser != null) conf.set(CONF_FILES_USER, filesUser);
    conf.setInt(CONF_FILES_MODE, filesMode);
    conf.setBoolean(CONF_CHECKSUM_VERIFY, verifyChecksum);
    conf.set(CONF_OUTPUT_ROOT, outputRoot.toString());
    conf.set(CONF_INPUT_ROOT, inputRoot.toString());
    conf.setInt("mapreduce.job.maps", mappers);

    // job.setMapSpeculativeExecution(false)
    conf.setBoolean("mapreduce.map.speculative", false);
    conf.setBoolean("mapreduce.reduce.speculative", false);
    conf.setBoolean("mapred.map.tasks.speculative.execution", false);
    conf.setBoolean("mapred.reduce.tasks.speculative.execution", false);

    Job job = new Job(conf);
    job.setJobName("ExportSnapshot");
    job.setJarByClass(ExportSnapshot.class);
    job.setMapperClass(ExportMapper.class);
View Full Code Here

   * @param newDesc new table descriptor to use
   * @deprecated Do not use; expensive call
   */
  @Deprecated
  public void setTableDesc(HTableDescriptor newDesc) {
    Configuration c = HBaseConfiguration.create();
    FileSystem fs;
    try {
      fs = FileSystem.get(c);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    FSTableDescriptors fstd =
      new FSTableDescriptors(fs, new Path(c.get(HConstants.HBASE_DIR)));
    try {
      fstd.add(newDesc);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
View Full Code Here

    if (outputRoot == null) {
      System.err.println("Destination file-system not provided.");
      printUsageAndExit();
    }

    Configuration conf = getConf();
    Path inputRoot = FSUtils.getRootDir(conf);
    FileSystem inputFs = FileSystem.get(conf);
    FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);

    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
View Full Code Here

    }
    // either dump using the HLogPrettyPrinter or split, depending on args
    if (args[0].compareTo("--dump") == 0) {
      HLogPrettyPrinter.run(Arrays.copyOfRange(args, 1, args.length));
    } else if (args[0].compareTo("--split") == 0) {
      Configuration conf = HBaseConfiguration.create();
      for (int i = 1; i < args.length; i++) {
        try {
          conf.set("fs.default.name", args[i]);
          conf.set("fs.defaultFS", args[i]);
          Path logPath = new Path(args[i]);
          split(conf, logPath);
        } catch (Throwable t) {
          t.printStackTrace(System.err);
          System.exit(-1);
View Full Code Here

    String nodeName = rss.getServerName().toString();
    this.memberRpcs = new ZKProcedureMemberRpcs(zkw,
        SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, nodeName);

    // read in the snapshot request configuration properties
    Configuration conf = rss.getConfiguration();
    long wakeMillis = conf.getLong(SNAPSHOT_REQUEST_WAKE_MILLIS_KEY, SNAPSHOT_REQUEST_WAKE_MILLIS_DEFAULT);
    long keepAlive = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
    int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT);

    // create the actual snapshot procedure member
    ThreadPoolExecutor pool = ProcedureMember.defaultPool(wakeMillis, keepAlive, opThreads, nodeName);
    this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder());
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.conf.Configuration

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.