Examples of org.apache.hadoop.fs.Path

org.apache.hadoop.fs.Path
Names a file or directory in a {@link FileSystem}. Path strings use slash as the directory separator. A path string is absolute if it begins with a slash.

   * @param hbi
   * @return
   * @throws IOException
   */
  private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
    Path regionDir = hbi.getHdfsRegionDir();
    FileSystem fs = regionDir.getFileSystem(getConf());
    FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
    for (FileStatus subdir : subDirs) {
      String columnfamily = subdir.getPath().getName();
      columns.add(columnfamily);
    }

View Full Code Here

   * @throws IOException
   */
  public void fixOrphanTables() throws IOException {
    if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {


      Path hbaseRoot = FSUtils.getRootDir(getConf());
      List<String> tmpList = new ArrayList<String>();
      tmpList.addAll(orphanTableDirs.keySet());
      HTableDescriptor[] htds = getHTableDescriptors(tmpList);
      Iterator<Entry<String, Set<String>>> iter = orphanTableDirs.entrySet().iterator();
      int j = 0;
      int numFailedCase = 0;
      while (iter.hasNext()) {
        Entry<String, Set<String>> entry = (Entry<String, Set<String>>) iter.next();
        String tableName = entry.getKey();
        LOG.info("Trying to fix orphan table error: " + tableName);
        if (j < htds.length) {
          if (tableName.equals(Bytes.toString(htds[j].getName()))) {
            HTableDescriptor htd = htds[j];
            LOG.info("fixing orphan table: " + tableName + " from cache");
            FSTableDescriptors.createTableDescriptor(
                hbaseRoot.getFileSystem(getConf()), hbaseRoot, htd, true);
            j++;
            iter.remove();
          }
        } else {
          if (fabricateTableInfo(tableName, entry.getValue())) {

View Full Code Here

   * This borrows code from MasterFileSystem.bootstrap()
   * 
   * @return an open .META. HRegion
   */
  private HRegion createNewRootAndMeta() throws IOException {
    Path rootdir = new Path(getConf().get(HConstants.HBASE_DIR));
    Configuration c = getConf();
    HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
    MasterFileSystem.setInfoFamilyCachingForRoot(false);
    HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
    MasterFileSystem.setInfoFamilyCachingForMeta(false);

View Full Code Here

      }
    }


    // we can rebuild, move old root and meta out of the way and start
    LOG.info("HDFS regioninfo's seems good.  Sidelining old .META.");
    Path backupDir = sidelineOldRootAndMeta();


    LOG.info("Creating new .META.");
    HRegion meta = createNewRootAndMeta();


    // populate meta

View Full Code Here

    return tablesInfo;
  }


  private Path getSidelineDir() throws IOException {
    if (sidelineDir == null) {
      Path hbaseDir = FSUtils.getRootDir(getConf());
      Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
      sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
          + startMillis);
    }
    return sidelineDir;
  }

View Full Code Here

   * regions should be bulk loaded back later on. If null, it is ignored.
   */
  Path sidelineRegionDir(FileSystem fs,
      String parentDir, HbckInfo hi) throws IOException {
    String tableName = Bytes.toString(hi.getTableName());
    Path regionDir = hi.getHdfsRegionDir();


    if (!fs.exists(regionDir)) {
      LOG.warn("No previous " + regionDir + " exists.  Continuing.");
      return null;
    }


    Path rootDir = getSidelineDir();
    if (parentDir != null) {
      rootDir = new Path(rootDir, parentDir);
    }
    Path sidelineTableDir= new Path(rootDir, tableName);
    Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
    fs.mkdirs(sidelineRegionDir);
    boolean success = false;
    FileStatus[] cfs =  fs.listStatus(regionDir);
    if (cfs == null) {
      LOG.info("Region dir is empty: " + regionDir);
    } else {
      for (FileStatus cf : cfs) {
        Path src = cf.getPath();
        Path dst =  new Path(sidelineRegionDir, src.getName());
        if (fs.isFile(src)) {
          // simple file
          success = fs.rename(src, dst);
          if (!success) {
            String msg = "Unable to rename file " + src +  " to " + dst;

View Full Code Here

   * Side line an entire table.
   */
  void sidelineTable(FileSystem fs, byte[] table, Path hbaseDir,
      Path backupHbaseDir) throws IOException {
    String tableName = Bytes.toString(table);
    Path tableDir = new Path(hbaseDir, tableName);
    if (fs.exists(tableDir)) {
      Path backupTableDir= new Path(backupHbaseDir, tableName);
      boolean success = fs.rename(tableDir, backupTableDir);
      if (!success) {
        throw new IOException("Failed to move  " + tableName + " from " 
            +  tableDir.getName() + " to " + backupTableDir.getName());
      }
    } else {
      LOG.info("No previous " + tableName +  " exists.  Continuing.");
    }
  }

View Full Code Here

  /**
   * @return Path to backup of original directory
   */
  Path sidelineOldRootAndMeta() throws IOException {
    // put current -ROOT- and .META. aside.
    Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
    FileSystem fs = hbaseDir.getFileSystem(getConf());
    Path backupDir = getSidelineDir();
    fs.mkdirs(backupDir);


    sidelineTable(fs, HConstants.ROOT_TABLE_NAME, hbaseDir, backupDir);
    try {
      sidelineTable(fs, HConstants.META_TABLE_NAME, hbaseDir, backupDir);
    } catch (IOException e) {
      LOG.error("Attempt to sideline meta failed, attempt to revert...", e);
      try {
        // move it back.
        sidelineTable(fs, HConstants.ROOT_TABLE_NAME, backupDir, hbaseDir);
        LOG.warn("... revert succeed.  -ROOT- and .META. still in "
            + "original state.");
      } catch (IOException ioe) {
        LOG.fatal("... failed to sideline root and meta and failed to restore "
            + "prevoius state.  Currently in inconsistent state.  To restore "
            + "try to rename -ROOT- in " + backupDir.getName() + " to " 
            + hbaseDir.getName() + ".", ioe);
      }
      throw e; // throw original exception
    }
    return backupDir;

View Full Code Here

  /**
   * Scan HDFS for all regions, recording their information into
   * regionInfoMap
   */
  public void loadHdfsRegionDirs() throws IOException, InterruptedException {
    Path rootDir = new Path(getConf().get(HConstants.HBASE_DIR));
    FileSystem fs = rootDir.getFileSystem(getConf());


    // list all tables from HDFS
    List<FileStatus> tableDirs = Lists.newArrayList();


    boolean foundVersionFile = false;

View Full Code Here

  private void preCheckPermission() throws IOException, AccessControlException {
    if (shouldIgnorePreCheckPermission()) {
      return;
    }


    Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
    FileSystem fs = hbaseDir.getFileSystem(getConf());
    User user = User.getCurrent();
    FileStatus[] files = fs.listStatus(hbaseDir);
    for (FileStatus file : files) {
      try {
        FSUtils.checkAccess(user, file, FsAction.WRITE);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.fs.Path

co.nubetech.hiho.common.HihoTestCase

co.nubetech.hiho.dedup.TestDedupJob

co.nubetech.hiho.job.TestDBQueryInputJobWithCluster

co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat

co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat

co.nubetech.hiho.merge.TestMergeJob

com.alimama.mdrill.utils.HadoopBaseUtils

com.alimama.mdrill.utils.IndexUtils

com.alimama.mdrill.utils.ZipUtils

com.cloudera.circus.test.TestXTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.