Package org.apache.hadoop.fs

Source Code of org.apache.hadoop.fs.FileSystem$GlobFilter

/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs;

import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;

import javax.security.auth.login.LoginException;

import org.apache.commons.logging.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.fs.FileSystem.Cache.Key;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.ReadOptions;
import org.apache.hadoop.io.WriteOptions;
import org.apache.hadoop.security.UserGroupInformation;

/****************************************************************
* An abstract base class for a fairly generic filesystem.  It
* may be implemented as a distributed filesystem, or as a "local"
* one that reflects the locally-connected disk.  The local version
* exists for small Hadoop instances and for testing.
*
* <p>
*
* All user code that may potentially use the Hadoop Distributed
* File System should be written to use a FileSystem object.  The
* Hadoop DFS is a multi-machine system that appears as a single
* disk.  It's useful because of its fault tolerance and potentially
* very large capacity.
*
* <p>
* The local implementation is {@link LocalFileSystem} and distributed
* implementation is DistributedFileSystem.
*****************************************************************/
public abstract class FileSystem extends Configured implements Closeable {

  public static final String FS_DEFAULT_NAME_KEY = "fs.default.name";

  public static final Log LOG = LogFactory.getLog(FileSystem.class);

  public static final Log LogForCollect = LogFactory.getLog(FileSystem.class
      .getName() + ".collect");

  /** FileSystem cache */
  private static final Cache CACHE = new Cache();

  /** The key this instance is stored under in the cache. */
  private Cache.Key key;

  /** Recording statistics per a FileSystem class */
  private static final Map<Class<? extends FileSystem>, Statistics>
    statisticsTable =
      new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
 
  /**
   * The statistics for this file system.
   */
  protected Statistics statistics;

  /**
   * A cache of files that should be deleted when filsystem is closed
   * or the JVM is exited.
   */
  private Set<Path> deleteOnExit = new TreeSet<Path>();

  /**
   * shutdownHook is true if we should close this FileSystem when
   * our shutdown hook (clientFinalizer) is invoked.
   */
  private boolean shutdownHook = false;

  /** Returns the configured filesystem implementation.*/
  public static FileSystem get(Configuration conf) throws IOException {
    return get(getDefaultUri(conf), conf);
  }
 
  /** Get the default filesystem URI from a configuration.
   * @param conf the configuration to access
   * @return the uri of the default filesystem
   */
  public static URI getDefaultUri(Configuration conf) {
    return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, "file:///")));
  }

  /** Set the default filesystem URI in a configuration.
   * @param conf the configuration to alter
   * @param uri the new default filesystem uri
   */
  public static void setDefaultUri(Configuration conf, URI uri) {
    conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
  }

  /** Set the default filesystem URI in a configuration.
   * @param conf the configuration to alter
   * @param uri the new default filesystem uri
   */
  public static void setDefaultUri(Configuration conf, String uri) {
    setDefaultUri(conf, URI.create(fixName(uri)));
  }

  /** Called after a new FileSystem instance is constructed.
   * @param name a uri whose authority section names the host, port, etc.
   *   for this FileSystem
   * @param conf the configuration
   */
  public void initialize(URI name, Configuration conf) throws IOException {
    statistics = getStatistics(name.getScheme(), getClass());   
  }

  /** Returns a URI whose scheme and authority identify this FileSystem.*/
  public abstract URI getUri();
 
  /** @deprecated call #getUri() instead.*/
  public String getName() { return getUri().toString(); }

  /** @deprecated call #get(URI,Configuration) instead. */
  public static FileSystem getNamed(String name, Configuration conf)
    throws IOException {
    return get(URI.create(fixName(name)), conf);
  }
 
  /** Update old-format filesystem names, for back-compatibility.  This should
   * eventually be replaced with a checkName() method that throws an exception
   * for old-format names. */
  private static String fixName(String name) {
    // convert old-format name to new-format name
    if (name.equals("local")) {         // "local" is now "file:///".
      LOG.warn("\"local\" is a deprecated filesystem name."
               +" Use \"file:///\" instead.");
      name = "file:///";
    } else if (name.indexOf('/')==-1) {   // unqualified is "hdfs://"
      LOG.warn("\""+name+"\" is a deprecated filesystem name."
               +" Use \"hdfs://"+name+"/\" instead.");
      name = "hdfs://"+name;
    }
    return name;
  }

  /**
   * Get the local file syste
   * @param conf the configuration to configure the file system with
   * @return a LocalFileSystem
   */
  public static LocalFileSystem getLocal(Configuration conf)
    throws IOException {
    return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
  }

  /** Returns the FileSystem for this URI's scheme and authority.  The scheme
   * of the URI determines a configuration property name,
   * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
   * The entire URI is passed to the FileSystem instance's initialize method.
   */
  public static FileSystem get(URI uri, Configuration conf) throws IOException {
    String scheme = uri.getScheme();
    String authority = uri.getAuthority();

    if (scheme == null) {                       // no scheme: use default FS
      return get(conf);
    }

    if (authority == null) {                       // no authority
      URI defaultUri = getDefaultUri(conf);
      if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
          && defaultUri.getAuthority() != null) {  // & default has authority
        return get(defaultUri, conf);              // return default
      }
    }

    String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
    if (conf.getBoolean(disableCacheName, false)) {
      return createFileSystem(uri, conf, null);
    }

    return CACHE.get(uri, conf);
  }

  /** Returns the FileSystem for this URI's scheme and authority.  The scheme
   * of the URI determines a configuration property name,
   * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
   * The entire URI is passed to the FileSystem instance's initialize method.
   * This always returns a new FileSystem object.
   */
  public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
    String scheme = uri.getScheme();
    String authority = uri.getAuthority();

    if (scheme == null) {                       // no scheme: use default FS
      return newInstance(conf);
    }

    if (authority == null) {                       // no authority
      URI defaultUri = getDefaultUri(conf);
      if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
          && defaultUri.getAuthority() != null) {  // & default has authority
        return newInstance(defaultUri, conf);              // return default
      }
    }
    return CACHE.getUnique(uri, conf);
  }

  /** Returns a unique configured filesystem implementation.
   * This always returns a new FileSystem object. */
  public static FileSystem newInstance(Configuration conf) throws IOException {
    return newInstance(getDefaultUri(conf), conf);
  }

  /**
   * Get a unique local file system object
   * @param conf the configuration to configure the file system with
   * @return a LocalFileSystem
   * This always returns a new FileSystem object.
   */
  public static LocalFileSystem newInstanceLocal(Configuration conf)
    throws IOException {
    return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
  }

  /**
   * ClientFinalizer is the class we use as a shutdown hook, called
   * when the JVM shuts down.  It will close all cached FileSystems that
   * have shutdownHook set.
   */
  private static class ClientFinalizer extends Thread {
    public void run() {
      try {
        CACHE.closeAll(new ShutdownSelect());
      } catch (IOException e) {
        LOG.info("FileSystem.closeAll() threw an exception:\n" + e);
      }
    }
  }
  private static final ClientFinalizer clientFinalizer = new ClientFinalizer();

  /**
   * Close all cached filesystems. Be sure those filesystems are not
   * used anymore.
   *
   * @throws IOException
   */
  public static void closeAll() throws IOException {
    CACHE.closeAll(new AllSelect());
  }

  /**
   * Close all cached filesystems that match the supplied ugi.
   */
  public static void closeAllForUGI(UserGroupInformation ugi) throws IOException {
    CACHE.closeAll(new UGISelect(ugi));
}

  /** Make sure that a path specifies a FileSystem. */
  public Path makeQualified(Path path) {
    checkPath(path);
    return path.makeQualified(this);
  }
   
  /** create a file with the provided permission
   * The permission of the file is set to be the provided permission as in
   * setPermission, not permission&~umask
   *
   * It is implemented using two RPCs. It is understood that it is inefficient,
   * but the implementation is thread-safe. The other option is to change the
   * value of umask in configuration to be 0, but it is not thread-safe.
   *
   * @param fs file system handle
   * @param file the name of the file to be created
   * @param permission the permission of the file
   * @return an output stream
   * @throws IOException
   */
  public static FSDataOutputStream create(FileSystem fs,
      Path file, FsPermission permission) throws IOException {
    // create the file with default permission
    FSDataOutputStream out = fs.create(file);
    // set its permission to the supplied one
    fs.setPermission(file, permission);
    return out;
  }

  /** create a directory with the provided permission
   * The permission of the directory is set to be the provided permission as in
   * setPermission, not permission&~umask
   *
   * @see #create(FileSystem, Path, FsPermission)
   *
   * @param fs file system handle
   * @param dir the name of the directory to be created
   * @param permission the permission of the directory
   * @return true if the directory creation succeeds; false otherwise
   * @throws IOException
   */
  public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission)
  throws IOException {
    // create the directory using the default permission
    boolean result = fs.mkdirs(dir);
    // set its permission to be the supplied one
    fs.setPermission(dir, permission);
    return result;
  }

  ///////////////////////////////////////////////////////////////
  // FileSystem
  ///////////////////////////////////////////////////////////////

  protected FileSystem() {
    super(null);
  }

  protected long getUniqueId() {
    if (key == null) {
      return 0;
    } else {
      return key.unique;
    }
  }
 
  /** Check that a Path belongs to this FileSystem. */
  protected void checkPath(Path path) {
    URI uri = path.toUri();
    if (uri.getScheme() == null)                // fs is relative
      return;
    String thisScheme = this.getUri().getScheme();
    String thatScheme = uri.getScheme();
    String thisAuthority = this.getUri().getAuthority();
    String thatAuthority = uri.getAuthority();
    //authority and scheme are not case sensitive
    if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match
      if (thisAuthority == thatAuthority ||       // & authorities match
          (thisAuthority != null &&
           thisAuthority.equalsIgnoreCase(thatAuthority)))
        return;

      if (thatAuthority == null &&                // path's authority is null
          thisAuthority != null) {                // fs has an authority
        URI defaultUri = getDefaultUri(getConf()); // & is the conf default
        if (thisScheme.equalsIgnoreCase(defaultUri.getScheme()) &&
            thisAuthority.equalsIgnoreCase(defaultUri.getAuthority()))
          return;
        try {                                     // or the default fs's uri
          defaultUri = get(getConf()).getUri();
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
        if (thisScheme.equalsIgnoreCase(defaultUri.getScheme()) &&
            thisAuthority.equalsIgnoreCase(defaultUri.getAuthority()))
          return;
      }
    }
    throw new IllegalArgumentException("Wrong FS: "+path+
                                       ", expected: "+this.getUri());
  }

  /**
   * Return an array containing hostnames, offset and size of
   * portions of the given file.  For a nonexistent
   * file or regions, null will be returned.
   *
   * This call is most helpful with DFS, where it returns
   * hostnames of machines that contain the given file.
   *
   * The FileSystem will simply return an elt containing 'localhost'.
   */
  public BlockLocation[] getFileBlockLocations(FileStatus file,
      long start, long len) throws IOException {
    if (file == null) {
      return null;
    }

    if ( (start<0) || (len < 0) ) {
      throw new IllegalArgumentException("Invalid start or len parameter");
    }

    if (file.getLen() < start) {
      return new BlockLocation[0];

    }
    String[] name = { "localhost:50010" };
    String[] host = { "localhost" };
    return new BlockLocation[] { new BlockLocation(name, host, 0, file.getLen()) };
  }
 
  /**
   * Opens an FSDataInputStream at the indicated Path.
   * @param f the file name to open
   * @param bufferSize the size of the buffer to be used.
   */
  public abstract FSDataInputStream open(Path f, int bufferSize)
    throws IOException;

  /**
   * Opens an FSDataInputStream at the indicated Path.
   *
   * @param f
   *          the file name to open
   * @param bufferSize
   *          the size of the buffer to be used.
   * @param options
   *          the read options passed by the user
   */
  public FSDataInputStream open(Path f, int bufferSize, ReadOptions options)
      throws IOException {
    throw new IOException("operation not supported for this filesystem");
  }
   
  /**
   * Opens an FSDataInputStream at the indicated Path.
   * @param f the file to open
   */
  public FSDataInputStream open(Path f) throws IOException {
    return open(f, getDefaultBufferSize());
  }

  /**
   * Opens an FSDataInputStream at the indicated Path.
   *
   * @param f
   *          the file to open
   * @param options
   *          the read options for this file
   */
  public FSDataInputStream open(Path f, ReadOptions options) throws IOException {
    return open(f, getDefaultBufferSize(), options);
  }

  /**
   * Opens an FSDataOutputStream at the indicated Path.
   * Files are overwritten by default.
   */
  public FSDataOutputStream create(Path f) throws IOException {
    return create(f, CreateOptions.writeOptions(true, null));
  }

  /**
   * Opens an FSDataOutputStream at the indicated Path.
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f, boolean overwrite) throws IOException {
    return create(f, CreateOptions.writeOptions(overwrite, null));
  }

  /**
   * Create an FSDataOutputStream at the indicated Path with write-progress
   * reporting.
   * Files are overwritten by default.
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f, Progressable progress) throws IOException {
    return create(f, CreateOptions.progress(progress));
  }

  /**
   * Opens an FSDataOutputStream at the indicated Path.
   * Files are overwritten by default.
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f, short replication)
    throws IOException {
    return create(f, CreateOptions.replicationFactor(replication));
  }

  /**
   * Opens an FSDataOutputStream at the indicated Path with write-progress
   * reporting.
   * Files are overwritten by default.
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f, short replication, Progressable progress)
    throws IOException {
    return create(f, CreateOptions.replicationFactor(replication), CreateOptions.progress(progress));
  }

   
  /**
   * Opens an FSDataOutputStream at the indicated Path.
   * @param f the file name to open
   * @param overwrite if a file with this name already exists, then if true,
   *   the file will be overwritten, and if false an error will be thrown.
   * @param bufferSize the size of the buffer to be used.
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f,
                                   boolean overwrite,
                                   int bufferSize
                                   ) throws IOException {
    return create(f, CreateOptions.writeOptions(overwrite, null), CreateOptions.bufferSize(bufferSize));
  }
   
  /**
   * Opens an FSDataOutputStream at the indicated Path with write-progress
   * reporting.
   * @param f the file name to open
   * @param overwrite if a file with this name already exists, then if true,
   *   the file will be overwritten, and if false an error will be thrown.
   * @param bufferSize the size of the buffer to be used.
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f,
                                   boolean overwrite,
                                   int bufferSize,
                                   Progressable progress
                                   ) throws IOException {
    return create(f, CreateOptions.writeOptions(overwrite, null),
        CreateOptions.bufferSize(bufferSize), CreateOptions.progress(progress));
  }
   
   
  /**
   * Opens an FSDataOutputStream at the indicated Path.
   * @param f the file name to open
   * @param overwrite if a file with this name already exists, then if true,
   *   the file will be overwritten, and if false an error will be thrown.
   * @param bufferSize the size of the buffer to be used.
   * @param replication required block replication for the file.
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f,
                                   boolean overwrite,
                                   int bufferSize,
                                   short replication,
                                   long blockSize
                                   ) throws IOException {
    return create(f, CreateOptions.writeOptions(overwrite, null),
        CreateOptions.bufferSize(bufferSize), CreateOptions.replicationFactor(replication),
        CreateOptions.blockSize(blockSize));
  }

  /**
   * Opens an FSDataOutputStream at the indicated Path with write-progress
   * reporting.
   * @param f the file name to open
   * @param overwrite if a file with this name already exists, then if true,
   *   the file will be overwritten, and if false an error will be thrown.
   * @param bufferSize the size of the buffer to be used.
   * @param replication required block replication for the file.
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f,
                                            boolean overwrite,
                                            int bufferSize,
                                            short replication,
                                            long blockSize,
                                            Progressable progress
                                            ) throws IOException {
    return this.create(f, CreateOptions.writeOptions(overwrite, null),
        CreateOptions.bufferSize(bufferSize), CreateOptions.replicationFactor(replication),
        CreateOptions.blockSize(blockSize), CreateOptions.progress(progress));
  }
 
  private Object getOption(Class<? extends CreateOptions> clazz, Object defaultValue,
      CreateOptions... opts) {
    CreateOptions createOptions = CreateOptions.getOpt(clazz, opts);
    if (createOptions != null)
      return createOptions.getValue();
    return defaultValue;
  }
 
  public FSDataOutputStream create(Path f, CreateOptions... opts) throws IOException {
    // Arguments Default values:
    FsPermission perms = (FsPermission) getOption(CreateOptions.Perms.class,
        FsPermission.getDefault(), opts);
    boolean overwrite = ((WriteOptions) getOption(WriteOptions.class, new WriteOptions(), opts))
        .getOverwrite();
    int bufferSize = (Integer) getOption(CreateOptions.BufferSize.class, getDefaultBufferSize(),
        opts);
    short replication = (Short) getOption(CreateOptions.ReplicationFactor.class,
        getDefaultReplication(), opts);
    long blockSize = (Long) getOption(CreateOptions.BlockSize.class, getDefaultBlockSize(), opts);
    boolean forceSync = ((WriteOptions) getOption(WriteOptions.class, new WriteOptions(), opts))
        .getForceSync();
    Progressable progress = (Progressable) getOption(CreateOptions.Progress.class, null, opts);

    if (forceSync)
      throw new IOException("Create force sync file is unsupported " + "for this filesystem"
          + this.getClass());

    return create(f, perms, overwrite, bufferSize, replication, blockSize, progress);
  }

  /**
   * Opens an FSDataOutputStream at the indicated Path with write-progress
   * reporting.
   * @param f the file name to open
   * @param permission
   * @param overwrite if a file with this name already exists, then if true,
   *   the file will be overwritten, and if false an error will be thrown.
   * @param bufferSize the size of the buffer to be used.
   * @param replication required block replication for the file.
   * @param blockSize
   * @param progress
   * @throws IOException
   * @see #setPermission(Path, FsPermission)
   */
  public abstract FSDataOutputStream create(Path f,
      FsPermission permission,
      boolean overwrite,
      int bufferSize,
      short replication,
      long blockSize,
      Progressable progress) throws IOException;

  /**
   * Opens an FSDataOutputStream at the indicated Path with write-progress
   * reporting.
   * @param f the file name to open
   * @param permission
   * @param overwrite if a file with this name already exists, then if true,
   *   the file will be overwritten, and if false an error will be thrown.
   * @param bufferSize the size of the buffer to be used.
   * @param replication required block replication for the file.
   * @param blockSize
   * @param bytesPerChecksum the number of data bytes per checksum
   * @param progress
   * @throws IOException
   * @see #setPermission(Path, FsPermission)
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f,
      FsPermission permission,
      boolean overwrite,
      int bufferSize,
      short replication,
      long blockSize,
      int bytesPerChecksum,
      Progressable progress) throws IOException {
    return create(f, CreateOptions.perms(permission), CreateOptions.writeOptions(overwrite, null),
        CreateOptions.bufferSize(bufferSize), CreateOptions.replicationFactor(replication),
        CreateOptions.blockSize(blockSize), CreateOptions.bytesPerChecksum(bytesPerChecksum),
        CreateOptions.progress(progress));
  }
 
  /**
   * @deprecated Use {@link #create(Path, CreateOptions...)} instead.
   */
  public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite,
      int bufferSize, short replication, long blockSize, int bytesPerChecksum,
      Progressable progress, boolean forceSync) throws IOException {
    return create(f, CreateOptions.perms(permission),
        CreateOptions.writeOptions(overwrite, forceSync), CreateOptions.bufferSize(bufferSize),
        CreateOptions.replicationFactor(replication), CreateOptions.blockSize(blockSize),
        CreateOptions.bytesPerChecksum(bytesPerChecksum), CreateOptions.progress(progress));
  }

  /**
   * Opens an FSDataOutputStream at the indicated Path with write-progress
   * reporting. Same as create(), except fails if parent directory doesn't
   * already exist.
   * @param f the file name to open
   * @param overwrite if a file with this name already exists, then if true,
   * the file will be overwritten, and if false an error will be thrown.
   * @param bufferSize the size of the buffer to be used.
   * @param replication required block replication for the file.
   * @param blockSize
   * @param progress
   * @throws IOException
   * @see #setPermission(Path, FsPermission)
   * @deprecated API only for 0.20-append
   */
   @Deprecated
   public FSDataOutputStream createNonRecursive(Path f,
       boolean overwrite,
       int bufferSize, short replication, long blockSize,
       Progressable progress) throws IOException {
     return this.createNonRecursive(f, FsPermission.getDefault(),
         overwrite, bufferSize, replication, blockSize, progress);
   }

   /**
    * Opens an FSDataOutputStream at the indicated Path with write-progress
    * reporting. Same as create(), except fails if parent directory doesn't
    * already exist.
    * @param f the file name to open
    * @param permission
    * @param overwrite if a file with this name already exists, then if true,
    * the file will be overwritten, and if false an error will be thrown.
    * @param bufferSize the size of the buffer to be used.
    * @param replication required block replication for the file.
    * @param blockSize
    * @param progress
    * @throws IOException
    * @see #setPermission(Path, FsPermission)
    * @deprecated API only for 0.20-append
    */
    @Deprecated
    public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
        boolean overwrite,
        int bufferSize, short replication, long blockSize,
        Progressable progress) throws IOException {
      throw new IOException("createNonRecursive unsupported for this filesystem"
          + this.getClass());
    }

  /**
   * @deprecated API only for 0.20-append
   */
  public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
      boolean overwrite,
      int bufferSize, short replication, long blockSize,
      Progressable progress, boolean forceSync, boolean doParallelWrites)
      throws IOException {
    return createNonRecursive(f, permission, overwrite, bufferSize,
        replication, blockSize, progress, forceSync, doParallelWrites,
        new WriteOptions());
  }

  /**
  * Opens an FSDataOutputStream at the indicated Path with write-progress
  * reporting. Same as create(), except fails if parent directory doesn't
  * already exist.
  * @param f the file name to open
  * @param permission
  * @param overwrite if a file with this name already exists, then if true,
  * the file will be overwritten, and if false an error will be thrown.
  * @param bufferSize the size of the buffer to be used.
  * @param replication required block replication for the file.
  * @param blockSize
  * @param progress
  * @param forceSync
  * @throws IOException
  * @see #setPermission(Path, FsPermission)
  * @deprecated API only for 0.20-append
  */
  @Deprecated
  public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
      boolean overwrite,
      int bufferSize, short replication, long blockSize,
      Progressable progress, boolean forceSync, boolean doParallelWrites,
      WriteOptions options)
    throws IOException {
    throw new IOException("createNonRecursive unsupported for this filesystem"
        + this.getClass());
  }

  /**
   * Creates the given Path as a brand-new zero-length file.  If
   * create fails, or if it already existed, return false.
   */
  public boolean createNewFile(Path f) throws IOException {
    if (exists(f)) {
      return false;
    } else {
      create(f, false, getDefaultBufferSize()).close();
      return true;
    }
  }

  /**
   * Append to an existing file (optional operation).
   * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null)
   * @param f the existing file to be appended.
   * @throws IOException
   */
  public FSDataOutputStream append(Path f) throws IOException {
    return append(f, getDefaultBufferSize(), null);
  }
  /**
   * Append to an existing file (optional operation).
   * Same as append(f, bufferSize, null).
   * @param f the existing file to be appended.
   * @param bufferSize the size of the buffer to be used.
   * @throws IOException
   */
  public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
    return append(f, bufferSize, null);
  }

  /**
   * Append to an existing file (optional operation).
   * @param f the existing file to be appended.
   * @param bufferSize the size of the buffer to be used.
   * @param progress for reporting progress if it is not null.
   * @throws IOException
   */
  public abstract FSDataOutputStream append(Path f, int bufferSize,
      Progressable progress) throws IOException;
 
  /**
   * Get replication.
   *
   * @deprecated Use getFileStatus() instead
   * @param src file name
   * @return file replication
   * @throws IOException
   */
  @Deprecated
  public short getReplication(Path src) throws IOException {
    return getFileStatus(src).getReplication();
  }

  /**
   * Set replication for an existing file.
   *
   * @param src file name
   * @param replication new replication
   * @throws IOException
   * @return true if successful;
   *         false if file does not exist or is a directory
   */
  public boolean setReplication(Path src, short replication)
    throws IOException {
    return true;
  }

  /**
   * hard link Path dst to Path src. Can take place on DFS.
   */
  public boolean hardLink(Path src, Path dst) throws IOException {
    throw new UnsupportedOperationException(getClass().getCanonicalName()
        + " does not support hard link");
  }

  public String[] getHardLinkedFiles(Path src) throws IOException {
    throw new UnsupportedOperationException(getClass().getCanonicalName()
        + " does not support hard link");
  }
  
  /**
   * Renames Path src to Path dst.  Can take place on local fs
   * or remote DFS.
   */
  public abstract boolean rename(Path src, Path dst) throws IOException;
   
  /** Delete a file. */
  /** @deprecated Use delete(Path, boolean) instead */ @Deprecated
  public abstract boolean delete(Path f) throws IOException;
 
  /** Delete a file.
   *
   * @param f the path to delete.
   * @param recursive if path is a directory and set to
   * true, the directory is deleted else throws an exception. In
   * case of a file the recursive can be set to either true or false.
   * @return  true if delete is successful else false.
   * @throws IOException
   */
  public abstract boolean delete(Path f, boolean recursive) throws IOException;

  /**
   * Delete a file. Underlying FileSystem needs to override this method if
   * it implements a trash logic. The default behavior is to assume this
   * FileSystem doesn't have trash and all delete() is skipTrash.
   *
   * @param f
   *          the path to delete.
   * @param recursive
   *          if path is a directory and set to true, the directory is deleted
   *          else throws an exception. In case of a file the recursive can be
   *          set to either true or false.
   * @param skipTrash
   *          make sure the file won't be moved to trash if set to true. If the
   *          parameter is false, it doesn't mean the file will be moved to
   *          trash, as the underlying implementation file system might
   *          determine the file shouldn't go to trash, no trash is implemented
   *          or it is disabled.
   * @return true if delete is successful else false.
   * @throws IOException
   */
  public boolean delete(Path f, boolean recursive, boolean skipTrash)
      throws IOException {
    return delete(f, recursive);
  }

  /**
   * Undelete a file from trash.
   *
   * By default, return false.
   *
   * @param f the path to undelete
   * @param userName name of the user whose trash will be searched, or
   * null for current user
   * @return true if undelete is successful else false.
   * @throws IOException
   */
  public boolean undelete(Path f, String userName) throws IOException {
    return false;
  }

  /**
   * Mark a path to be deleted when FileSystem is closed.
   * When the JVM shuts down,
   * all FileSystem objects will be closed automatically.
   * Then,
   * the marked path will be deleted as a result of closing the FileSystem.
   *
   * The path has to exist in the file system.
   *
   * @param f the path to delete.
   * @return  true if deleteOnExit is successful, otherwise false.
   * @throws IOException
   */
  public boolean deleteOnExit(Path f) throws IOException {
    if (!exists(f)) {
      return false;
    }
    synchronized (deleteOnExit) {
      deleteOnExit.add(f);
    }
    return true;
  }

  /**
   * Delete all files that were marked as delete-on-exit. This recursively
   * deletes all files in the specified paths.
   */
  protected void processDeleteOnExit() {
    synchronized (deleteOnExit) {
      for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
        Path path = iter.next();
        try {
          delete(path, true);
        }
        catch (IOException e) {
          LOG.info("Ignoring failure to deleteOnExit for path " + path);
        }
        iter.remove();
      }
    }
  }
 
  /** Check if exists.
   * @param f source file
   */
  public boolean exists(Path f) throws IOException {
    try {
      return getFileStatus(f) != null;
    } catch (FileNotFoundException e) {
      return false;
    }
  }

  /** True iff the named path is a directory. */
  /** @deprecated Use getFileStatus() instead */ @Deprecated
  public boolean isDirectory(Path f) throws IOException {
    try {
      return getFileStatus(f).isDir();
    } catch (FileNotFoundException e) {
      return false;               // f does not exist
    }
  }

  /** True iff the named path is a regular file. */
  public boolean isFile(Path f) throws IOException {
    try {
      return !getFileStatus(f).isDir();
    } catch (FileNotFoundException e) {
      return false;               // f does not exist
    }
  }
   
  /** The number of bytes in a file. */
  /** @deprecated Use getFileStatus() instead */ @Deprecated
  public long getLength(Path f) throws IOException {
    return getFileStatus(f).getLen();
  }
   
  /** Return the {@link ContentSummary} of a given {@link Path}. */
  public ContentSummary getContentSummary(Path f) throws IOException {
    FileStatus status = getFileStatus(f);
    if (!status.isDir()) {
      // f is a file
      return new ContentSummary(status.getLen(), 1, 0);
    }
    // f is a directory
    long[] summary = {0, 0, 1};
    for(FileStatus s : listStatus(f)) {
      ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) :
                                     new ContentSummary(s.getLen(), 1, 0);
      summary[0] += c.getLength();
      summary[1] += c.getFileCount();
      summary[2] += c.getDirectoryCount();
    }
    return new ContentSummary(summary[0], summary[1], summary[2]);
  }

  private final static PathFilter DEFAULT_FILTER = new PathFilter() {
      public boolean accept(Path file) {
        return true;
      }    
    };
   
  /**
   * List the statuses of the files/directories in the given path if the path is
   * a directory.
   *
   * @param f
   *          given path
   * @return the statuses of the files/directories in the given patch
   * @throws IOException
   */
  public abstract FileStatus[] listStatus(Path f) throws IOException;
   
  /**
   * List the statuses of the files/directories in the given path if the path is
   * a directory.
   * Return the file's status and block locations If the path is a file.
   *
   * If a returned status is a file, it contains the file's block locations.
   *
   * @param f is the path
   *
   * @return an iterator that traverses statuses of the files/directories
   *         in the given path
   *
   * @throws FileNotFoundException If <code>f</code> does not exist
   * @throws IOException If an I/O error occurred
   */
  @Deprecated
  public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
  throws FileNotFoundException, IOException {
    return listLocatedStatus(f, DEFAULT_FILTER);
  }

  /**
   * Listing a directory
   * The returned results include its block location if it is a file
   * The results are filtered by the given path filter
   * @param f a path
   * @param filter a path filter
   * @return an iterator that traverses statuses of the files/directories
   *         in the given path
   * @throws FileNotFoundException if <code>f</code> does not exist
   * @throws IOException if any I/O error occurred
   */
  @Deprecated
  public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f,
      final PathFilter filter)
  throws FileNotFoundException, IOException {
    return new RemoteIterator<LocatedFileStatus>() {
      private final FileStatus[] stats;
      private int i = 0;

      { // initializer
        stats = listStatus(f, filter);
        if (stats == null) {
          throw new FileNotFoundException( "File " + f + " does not exist.");
        }
      }
     
      @Override
      public boolean hasNext() {
        return i<stats.length;
      }

      @Override
      public LocatedFileStatus next() throws IOException {
        if (!hasNext()) {
          throw new NoSuchElementException("No more entry in " + f);
        }
        FileStatus result = stats[i++];
        BlockLocation[] locs = result.isDir() ? null :
            getFileBlockLocations(result, 0, result.getLen());
        return new LocatedFileStatus(result, locs);
      }
    };
  }

  /**
   * List the statuses of the files/directories in the given path if the path is
   * a directory.
   * Return the file's status, blocks and locations if the path is a file.
   *
   * If a returned status is a file, it contains the file's blocks and locations.
   *
   * @param f is the path
   *
   * @return an iterator that traverses statuses of the files/directories
   *         in the given path
   *
   * @throws FileNotFoundException If <code>f</code> does not exist
   * @throws IOException If an I/O error occurred
   */
  public RemoteIterator<LocatedBlockFileStatus> listLocatedBlockStatus(
      final Path f) throws FileNotFoundException, IOException {
    return listLocatedBlockStatus(f, DEFAULT_FILTER);
  }

  /**
   * Listing a directory
   * The returned results include its blocks and locations if it is a file
   * The results are filtered by the given path filter
   * @param f a path
   * @param filter a path filter
   * @return an iterator that traverses statuses of the files/directories
   *         in the given path
   * @throws FileNotFoundException if <code>f</code> does not exist
   * @throws IOException if any I/O error occurred
   */
  public RemoteIterator<LocatedBlockFileStatus> listLocatedBlockStatus(
      final Path f,
      final PathFilter filter)
  throws FileNotFoundException, IOException {
    return new RemoteIterator<LocatedBlockFileStatus>() {
      private final FileStatus[] stats;
      private int i = 0;

      { // initializer
        stats = listStatus(f, filter);
        if (stats == null) {
          throw new FileNotFoundException( "File " + f + " does not exist.");
        }
      }
     
      @Override
      public boolean hasNext() {
        return i<stats.length;
      }

      @Override
      public LocatedBlockFileStatus next() throws IOException {
        if (!hasNext()) {
          throw new NoSuchElementException("No more entry in " + f);
        }
        FileStatus result = stats[i++];
        BlockAndLocation[] locs = null;
        if (!result.isDir()) {
          String[] name = { "localhost:50010" };
          String[] host = { "localhost" };
         
          // create a dummy blockandlocation
          locs = new BlockAndLocation[] {
              new BlockAndLocation(0L, 0L, name, host,
                  new String[0], 0, result.getLen(), false) };
        }
        return new LocatedBlockFileStatus(result, locs, false);
      }
    };
  }

  /*
   * Filter files/directories in the given path using the user-supplied path
   * filter. Results are added to the given array <code>results</code>.
   */
  private void listStatus(ArrayList<FileStatus> results, Path f,
      PathFilter filter) throws IOException {
    FileStatus listing[] = listStatus(f);
    if (listing == null) {
      throw new FileNotFoundException("File " + f + " does not exist");
    }
    for (int i = 0; i < listing.length; i++) {
      if (filter.accept(listing[i].getPath())) {
        results.add(listing[i]);
      }
    }
  }

  /**
   * @return an iterator over the corrupt files under the given path
   * (may contain duplicates if a file has more than one corrupt block)
   * @throws IOException
   */
  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
    throws IOException {
    throw new UnsupportedOperationException(getClass().getCanonicalName() +
                                            " does not support" +
                                            " listCorruptFileBlocks");
  }

  /**
   * Filter files/directories in the given path using the user-supplied path
   * filter.
   *
   * @param f
   *          a path name
   * @param filter
   *          the user-supplied path filter
   * @return an array of FileStatus objects for the files under the given path
   *         after applying the filter
   * @throws IOException
   *           if encounter any problem while fetching the status
   *           FileNotFoundException is thrown if path f doesn't exist
   */
  public FileStatus[] listStatus(Path f, PathFilter filter) throws IOException {
    ArrayList<FileStatus> results = new ArrayList<FileStatus>();
    listStatus(results, f, filter);
    return results.toArray(new FileStatus[results.size()]);
  }

  /**
   * Filter files/directories in the given list of paths using default
   * path filter.
   *
   * @param files
   *          a list of paths
   * @return a list of statuses for the files under the given paths after
   *         applying the filter default Path filter. If one or more of the files
   *         don't exist, there won't be FileNotFoundException thrown.
   *         Just no FileStatus will be added to the result array for that path.
   *        
   * @exception IOException
   */
  public FileStatus[] listStatus(Path[] files)
      throws IOException {
    return listStatus(files, DEFAULT_FILTER);
  }

  /**
   * Filter files/directories in the given list of paths using user-supplied
   * path filter.
   * If one of the user supplied directory does not exist, the method silently
   * skips it and continues with the remaining directories.
   *
   * @param files
   *          a list of paths
   * @param filter
   *          the user-supplied path filter
   * @return a list of statuses for the files under the given paths after
   *         applying the filter
   * @exception IOException
   */
  public FileStatus[] listStatus(Path[] files, PathFilter filter)
      throws IOException {
    ArrayList<FileStatus> results = new ArrayList<FileStatus>();
    for (int i = 0; i < files.length; i++) {
      try {
        listStatus(results, files[i], filter);
      } catch (FileNotFoundException e) {
        LOG.info("Parent path doesn't exist: " + e.getMessage());
      }
    }
    return results.toArray(new FileStatus[results.size()]);
  }

  /**
   * <p>Return all the files that match filePattern and are not checksum
   * files. Results are sorted by their names.
   *
   * <p>
   * A filename pattern is composed of <i>regular</i> characters and
   * <i>special pattern matching</i> characters, which are:
   *
   * <dl>
   <dd>
   *   <dl>
   *    <p>
   *    <dt> <tt> ? </tt>
   *    <dd> Matches any single character.
   *
   *    <p>
   *    <dt> <tt> * </tt>
   *    <dd> Matches zero or more characters.
   *
   *    <p>
   *    <dt> <tt> [<i>abc</i>] </tt>
   *    <dd> Matches a single character from character set
   *     <tt>{<i>a,b,c</i>}</tt>.
   *
   *    <p>
   *    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
   *    <dd> Matches a single character from the character range
   *     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
   *     lexicographically less than or equal to character <tt><i>b</i></tt>.
   *
   *    <p>
   *    <dt> <tt> [^<i>a</i>] </tt>
   *    <dd> Matches a single character that is not from character set or range
   *     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
   *     immediately to the right of the opening bracket.
   *
   *    <p>
   *    <dt> <tt> \<i>c</i> </tt>
   *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
   *
   *    <p>
   *    <dt> <tt> {ab,cd} </tt>
   *    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
   *   
   *    <p>
   *    <dt> <tt> {ab,c{de,fh}} </tt>
   *    <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
   *
   *   </dl>
   </dd>
   * </dl>
   *
   * @param pathPattern a regular expression specifying a pth pattern

   * @return an array of paths that match the path pattern
   * @throws IOException
   */
  public FileStatus[] globStatus(Path pathPattern) throws IOException {
    return globStatus(pathPattern, DEFAULT_FILTER);
  }
 
  /**
   * Return an array of FileStatus objects whose path names match pathPattern
   * and is accepted by the user-supplied path filter. Results are sorted by
   * their path names.
   * Return null if pathPattern has no glob and the path does not exist.
   * Return an empty array if pathPattern has a glob and no path matches it.
   *
   * @param pathPattern
   *          a regular expression specifying the path pattern
   * @param filter
   *          a user-supplied path filter
   * @return an array of FileStatus objects
   * @throws IOException if any I/O error occurs when fetching file status
   */
  public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
      throws IOException {
    String filename = pathPattern.toUri().getPath();
    List<String> filePatterns = GlobExpander.expand(filename);
    if (filePatterns.size() == 1) {
      return globStatusInternal(pathPattern, filter);
    } else {
      List<FileStatus> results = new ArrayList<FileStatus>();
      for (String filePattern : filePatterns) {
        FileStatus[] files = globStatusInternal(new Path(filePattern), filter);
        for (FileStatus file : files) {
          results.add(file);
        }
      }
      return results.toArray(new FileStatus[results.size()]);
    }
  }

  private FileStatus[] globStatusInternal(Path pathPattern, PathFilter filter)
      throws IOException {
    Path[] parents = new Path[1];
    int level = 0;
    String filename = pathPattern.toUri().getPath();
   
    // path has only zero component
    if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
      return getFileStatus(new Path[]{pathPattern});
    }

    // path has at least one component
    String[] components = filename.split(Path.SEPARATOR);
    // get the first component
    if (pathPattern.isAbsolute()) {
      parents[0] = new Path(Path.SEPARATOR);
      level = 1;
    } else {
      parents[0] = new Path(Path.CUR_DIR);
    }

    // glob the paths that match the parent path, i.e., [0, components.length-1]
    boolean[] hasGlob = new boolean[]{false};
    Path[] parentPaths = globPathsLevel(parents, components, level, hasGlob);
    FileStatus[] results;
    if (parentPaths == null || parentPaths.length == 0) {
      results = null;
    } else {
      // Now work on the last component of the path
      GlobFilter fp = new GlobFilter(components[components.length - 1], filter);
      if (fp.hasPattern()) { // last component has a pattern
        // list parent directories and then glob the results
        results = listStatus(parentPaths, fp);
        hasGlob[0] = true;
      } else { // last component does not have a pattern
        // get all the path names
        ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length);
        for (int i = 0; i < parentPaths.length; i++) {
          parentPaths[i] = new Path(parentPaths[i],
            components[components.length - 1]);
          if (fp.accept(parentPaths[i])) {
            filteredPaths.add(parentPaths[i]);
          }
        }
        // get all their statuses
        results = getFileStatus(
            filteredPaths.toArray(new Path[filteredPaths.size()]));
      }
    }

    // Decide if the pathPattern contains a glob or not
    if (results == null) {
      if (hasGlob[0]) {
        results = new FileStatus[0];
      }
    } else {
      if (results.length == 0 ) {
        if (!hasGlob[0]) {
          results = null;
        }
      } else {
        Arrays.sort(results);
      }
    }
    return results;
  }

  /*
   * For a path of N components, return a list of paths that match the
   * components [<code>level</code>, <code>N-1</code>].
   */
  private Path[] globPathsLevel(Path[] parents, String[] filePattern,
      int level, boolean[] hasGlob) throws IOException {
    if (level == filePattern.length - 1)
      return parents;
    if (parents == null || parents.length == 0) {
      return null;
    }
    GlobFilter fp = new GlobFilter(filePattern[level]);
    if (fp.hasPattern()) {
      parents = FileUtil.stat2Paths(listStatus(parents, fp));
      hasGlob[0] = true;
    } else {
      for (int i = 0; i < parents.length; i++) {
        parents[i] = new Path(parents[i], filePattern[level]);
      }
    }
    return globPathsLevel(parents, filePattern, level + 1, hasGlob);
  }

  /* A class that could decide if a string matches the glob or not */
  static class GlobFilter implements PathFilter {
    private PathFilter userFilter = DEFAULT_FILTER;
    private Pattern regex;
    private boolean hasPattern = false;
     
    /** Default pattern character: Escape any special meaning. */
    private static final char  PAT_ESCAPE = '\\';
    /** Default pattern character: Any single character. */
    private static final char  PAT_ANY = '.';
    /** Default pattern character: Character set close. */
    private static final char  PAT_SET_CLOSE = ']';
     
    GlobFilter() {
    }
     
    GlobFilter(String filePattern) throws IOException {
      setRegex(filePattern);
    }
     
    GlobFilter(String filePattern, PathFilter filter) throws IOException {
      userFilter = filter;
      setRegex(filePattern);
    }
     
    private boolean isJavaRegexSpecialChar(char pChar) {
      return pChar == '.' || pChar == '$' || pChar == '(' || pChar == ')' ||
             pChar == '|' || pChar == '+';
    }
    void setRegex(String filePattern) throws IOException {
      int len;
      int setOpen;
      int curlyOpen;
      boolean setRange;

      StringBuilder fileRegex = new StringBuilder();

      // Validate the pattern
      len = filePattern.length();
      if (len == 0)
        return;

      setOpen = 0;
      setRange = false;
      curlyOpen = 0;

      for (int i = 0; i < len; i++) {
        char pCh;
         
        // Examine a single pattern character
        pCh = filePattern.charAt(i);
        if (pCh == PAT_ESCAPE) {
          fileRegex.append(pCh);
          i++;
          if (i >= len)
            error("An escaped character does not present", filePattern, i);
          pCh = filePattern.charAt(i);
        } else if (isJavaRegexSpecialChar(pCh)) {
          fileRegex.append(PAT_ESCAPE);
        } else if (pCh == '*') {
          fileRegex.append(PAT_ANY);
          hasPattern = true;
        } else if (pCh == '?') {
          pCh = PAT_ANY;
          hasPattern = true;
        } else if (pCh == '{') {
          fileRegex.append('(');
          pCh = '(';
          curlyOpen++;
          hasPattern = true;
        } else if (pCh == ',' && curlyOpen > 0) {
          fileRegex.append(")|");
          pCh = '(';
        } else if (pCh == '}' && curlyOpen > 0) {
          // End of a group
          curlyOpen--;
          fileRegex.append(")");
          pCh = ')';
        } else if (pCh == '[' && setOpen == 0) {
          setOpen++;
          hasPattern = true;
        } else if (pCh == '^' && setOpen > 0) {
        } else if (pCh == '-' && setOpen > 0) {
          // Character set range
          setRange = true;
        } else if (pCh == PAT_SET_CLOSE && setRange) {
          // Incomplete character set range
          error("Incomplete character set range", filePattern, i);
        } else if (pCh == PAT_SET_CLOSE && setOpen > 0) {
          // End of a character set
          if (setOpen < 2)
            error("Unexpected end of set", filePattern, i);
          setOpen = 0;
        } else if (setOpen > 0) {
          // Normal character, or the end of a character set range
          setOpen++;
          setRange = false;
        }
        fileRegex.append(pCh);
      }
       
      // Check for a well-formed pattern
      if (setOpen > 0 || setRange || curlyOpen > 0) {
        // Incomplete character set or character range
        error("Expecting set closure character or end of range, or }",
            filePattern, len);
      }
      regex = Pattern.compile(fileRegex.toString());
    }
     
    boolean hasPattern() {
      return hasPattern;
    }
     
    public boolean accept(Path path) {
      return accept(path.getName()) && userFilter.accept(path);
    }
   
    boolean accept(String pathName) {
      return regex.matcher(pathName).matches();
    }
     
    private void error(String s, String pattern, int pos) throws IOException {
      throw new IOException("Illegal file pattern: "
                            +s+ " for glob "+ pattern + " at " + pos);
    }
  }

  /**
   * Return the current user's home directory in this filesystem.  The
   * default implementation returns "/user/$USER/".
   * @return Path for the current user's home directory
   */
  public Path getHomeDirectory() {
    return getHomeDirectory(null);
  }

  /**
   * Return the home directory for a given user in this filesystem.
   * @param userName name of the user or null for default user.name
   * @return Path for the supplied userName's home directory.
   */
  public Path getHomeDirectory(String userName) {
    if (userName == null)
      userName = System.getProperty("user.name");
    return new Path("/user/"+ userName).makeQualified(this);
  }

  /**
   * Set the current working directory for the given file system. All relative
   * paths will be resolved relative to it.
   *
   * @param new_dir
   */
  public abstract void setWorkingDirectory(Path new_dir);
   
  /**
   * Get the current working directory for the given file system
   * @return the directory pathname
   */
  public abstract Path getWorkingDirectory();

  /**
   * Call {@link #mkdirs(Path, FsPermission)} with default permission.
   */
  public boolean mkdirs(Path f) throws IOException {
    return mkdirs(f, FsPermission.getDefault());
  }

  /**
   * Make the given file and all non-existent parents into
   * directories. Has the semantics of Unix 'mkdir -p'.
   * Existence of the directory hierarchy is not an error.
   */
  public abstract boolean mkdirs(Path f, FsPermission permission
      ) throws IOException;

  /**
   * The src file is on the local disk.  Add it to FS at
   * the given dst name and the source is kept intact afterwards
   */
  @Deprecated
  public void copyFromLocalFile(Path src, Path dst)
    throws IOException {
    copyFromLocalFile(false, true, false, src, dst);
  }

  /**
   * The src files is on the local disk.  Add it to FS at
   * the given dst name, removing the source afterwards.
   */
  public void moveFromLocalFile(Path[] srcs, Path dst)
    throws IOException {
    copyFromLocalFile(true, true, false, srcs, dst);
  }

  /**
   * The src file is on the local disk.  Add it to FS at
   * the given dst name, removing the source afterwards.
   */
  public void moveFromLocalFile(Path src, Path dst)
    throws IOException {
    copyFromLocalFile(true, true, false, src, dst);
  }

  /**
   * The src file is on the local disk.  Add it to FS at
   * the given dst name.
   * delSrc indicates if the source should be removed
   */
  @Deprecated
  public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
    throws IOException {
    copyFromLocalFile(delSrc, true, src, dst);
  }
 
  /**
   * The src files are on the local disk.  Add it to FS at
   * the given dst name.
   * delSrc indicates if the source should be removed
   */
  @Deprecated
  public void copyFromLocalFile(boolean delSrc, boolean overwrite,
                                Path[] srcs, Path dst)
    throws IOException {
    copyFromLocalFile(delSrc, overwrite, false, srcs, dst);
  }
 
  /**
   * copy a list of files from local to a directory in this file system
   *
   * @param delSrc if source should be deleted
   * @param overwrite if destination should be overwritten
   * @param validate if copied destination should be validated against source
   * @param srcs a list of source files
   * @param dst destination
   * @throws IOException
   */
  public void copyFromLocalFile(boolean delSrc, boolean overwrite,
      boolean validate, Path[] srcs, Path dst) throws IOException {
    Configuration conf = getConf();
    FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite,
        validate, conf);
  }
 
  /**
   * The src file is on the local disk.  Add it to FS at
   * the given dst name.
   * delSrc indicates if the source should be removed
   */
  @Deprecated
  public void copyFromLocalFile(boolean delSrc, boolean overwrite,
                                Path src, Path dst)
    throws IOException {
    copyFromLocalFile(delSrc, overwrite, false, src, dst);
  }

  /**
   * copy a file from local to a file in this file system
   *
   * @param delSrc if source should be deleted
   * @param overwrite if destination should be overwritten
   * @param validate if copied destination should be validated against source
   * @param src source file
   * @param dst destination path
   * @throws IOException
   */
  public void copyFromLocalFile(boolean delSrc, boolean overwrite,
      boolean validate, Path src, Path dst)
    throws IOException {
    Configuration conf = getConf();
    FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite,
        validate, conf);
  }
   
  /**
   * The src file is under FS, and the dst is on the local disk.
   * Copy it from FS control to the local dst name.
   */
  @Deprecated
  public void copyToLocalFile(Path src, Path dst) throws IOException {
    copyToLocalFile(false, false, src, dst);
  }
   
  /**
   * The src file is under FS, and the dst is on the local disk.
   * Copy it from FS control to the local dst name.
   * Remove the source afterwards
   */
  public void moveToLocalFile(Path src, Path dst) throws IOException {
    copyToLocalFile(true, false, src, dst);
  }

  /**
   * The src file is under FS, and the dst is on the local disk.
   * Copy it from FS control to the local dst name.
   * delSrc indicates if the src will be removed or not.
   */  
  @Deprecated
  public void copyToLocalFile(boolean delSrc, Path src, Path dst)
    throws IOException {
    copyToLocalFile(delSrc, false, src, dst);
  }
 
  /**
   * Copy a file from this file system to local
   *
   * @param delSrc if source should be deleted
   * @param validate if copied destination should be validated against source
   * @param src source file
   * @param dst destination file
   * @throws IOException if error occurs
   */
  public void copyToLocalFile(boolean delSrc, boolean validate,
      Path src, Path dst)
  throws IOException {
    FileUtil.copy(this, src, getLocal(getConf()), dst, delSrc,
        validate, getConf());
  }

  /**
   * Returns a local File that the user can write output to.  The caller
   * provides both the eventual FS target name and the local working
   * file.  If the FS is local, we write directly into the target.  If
   * the FS is remote, we write into the tmp local area.
   */
  public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
    throws IOException {
    return tmpLocalFile;
  }

  /**
   * Called when we're all done writing to the target.  A local FS will
   * do nothing, because we've written to exactly the right place.  A remote
   * FS will copy the contents of tmpLocalFile to the correct target at
   * fsOutputFile.
   */
  public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
    throws IOException {
    moveFromLocalFile(tmpLocalFile, fsOutputFile);
  }

  /**
   * No more filesystem operations are needed.  Will
   * release any held locks.
   */
  public void close() throws IOException {
    // delete all files that were marked as delete-on-exit.
    processDeleteOnExit();
    CACHE.remove(this);
  }

  /**
   * Fetch the list of files that have been open longer than a
   * specified amount of time.
   * @param prefix path prefix specifying subset of files to examine
   * @param millis select files that have been open longer that this
   * @param start where to start searching in the case of subsequent calls, or null
   * @return array of OpenFileInfo objects
   * @throw IOException
   */
  public OpenFileInfo[] iterativeGetOpenFiles(Path prefix, int millis, String start)
    throws IOException {
    throw new UnsupportedOperationException(
      getClass().getCanonicalName() + " does not support iterativeGetOpenFiles");
  }

  /** Return the total size of all files in the filesystem.*/
  public long getUsed() throws IOException{
    long used = 0;
    FileStatus[] files = listStatus(new Path("/"));
    for(FileStatus file:files){
      used += file.getLen();
    }
    return used;
  }

  /**
   * Get the block size for a particular file.
   * @param f the filename
   * @return the number of bytes in a block
   */
  /** @deprecated Use getFileStatus() instead */ @Deprecated
  public long getBlockSize(Path f) throws IOException {
    return getFileStatus(f).getBlockSize();
  }
   
  /** Return the number of bytes that large input files should be optimally
   * be split into to minimize i/o time. */
  public long getDefaultBlockSize() {
    // default to 32MB: large enough to minimize the impact of seeks
    return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
  }
 
  private int getDefaultBufferSize(){
    return getConf().getInt("io.file.buffer.size", 4096);
  }
   
  /**
   * Get the default replication.
   */
  public short getDefaultReplication() { return 1; }

  /**
   * Return a file status object that represents the path.
   * @param f The path we want information from
   * @return a FileStatus object
   * @throws FileNotFoundException when the path does not exist;
   *         IOException see specific implementation
   */
  public abstract FileStatus getFileStatus(Path f) throws IOException;

  /**
   * Get the checksum of a file.
   *
   * @param f The file path
   * @return The file checksum.  The default return value is null,
   *  which indicates that no checksum algorithm is implemented
   *  in the corresponding FileSystem.
   */
  public FileChecksum getFileChecksum(Path f) throws IOException {
    return null;
  }

  /**
   * Get the CRC checksum of a file.
   *
   * @param f The file path
   * @return The file checksum.
   */
  public int getFileCrc(Path f) throws IOException {
    throw new IOException("Not implemented");
  }

 
  /**
   * Set the verify checksum flag. This is only applicable if the
   * corresponding FileSystem supports checksum. By default doesn't do anything.
   * @param verifyChecksum
   */
  public void setVerifyChecksum(boolean verifyChecksum) {
    //doesn't do anything
  }

  /**
   * Removes data from OS buffers after every read. This is only applicable
   * if the FileSystem/OS supports clearing OS buffers. By default, does
   * not do anything.
   * @param clearOsBuffer Set to true if every read from a file also removes
   * the data from the OS buffer cache.
   */
  public void clearOsBuffer(boolean clearOsBuffer) {
    //doesn't do anything
  }

  /**
   * Return a list of file status objects that corresponds to the list of paths
   * excluding those non-existent paths.
   *
   * @param paths
   *          the list of paths we want information from
   * @return a list of FileStatus objects
   * @throws IOException
   *           see specific implementation
   */
  private FileStatus[] getFileStatus(Path[] paths) throws IOException {
    if (paths == null) {
      return null;
    }
    ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
    for (int i = 0; i < paths.length; i++) {
      try {
        results.add(getFileStatus(paths[i]));
      } catch (FileNotFoundException e) { // do nothing
      }
    }
    return results.toArray(new FileStatus[results.size()]);
  }

  /**
   * Set permission of a path.
   * @param p
   * @param permission
   */
  public void setPermission(Path p, FsPermission permission
      ) throws IOException {
  }

  /**
   * Set owner of a path (i.e. a file or a directory).
   * The parameters username and groupname cannot both be null.
   * @param p The path
   * @param username If it is null, the original username remains unchanged.
   * @param groupname If it is null, the original groupname remains unchanged.
   */
  public void setOwner(Path p, String username, String groupname
      ) throws IOException {
  }

  /**
   * Set access time of a file
   * @param p The path
   * @param mtime Set the modification time of this file.
   *              The number of milliseconds since Jan 1, 1970.
   *              A value of -1 means that this call should not set modification time.
   * @param atime Set the access time of this file.
   *              The number of milliseconds since Jan 1, 1970.
   *              A value of -1 means that this call should not set access time.
   */
  public void setTimes(Path p, long mtime, long atime
      ) throws IOException {
  }

  private static FileSystem createFileSystem(URI uri, Configuration conf, Key key
      ) throws IOException {
    conf = ClientConfigurationUtil.mergeConfiguration(uri, conf);
    Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
    if (clazz == null) {
      throw new IOException("No FileSystem for scheme: " + uri.getScheme());
    }
    FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
    if (key != null) {
      fs.key = key;
    }
    fs.initialize(uri, conf);
    return fs;
  }

  /** Caching FileSystem objects */
  static class Cache {
    private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
    private final Set<Key> pending = new HashSet<Key>();

    /**
     * shutdownHookCount is the number of open FileSystems
     * with shutdownHook set.
     */
    private static int shutdownHookCount = 0;

    /** A variable that makes all objects in the cache unique */
    private static AtomicLong unique = new AtomicLong(1);

    FileSystem get(URI uri, Configuration conf) throws IOException{
      Key key = new Key(uri, conf);
      return getInternal(uri, conf, key);
    }

    /** The objects inserted into the cache using this method are all unique */
    FileSystem getUnique(URI uri, Configuration conf) throws IOException{
      Key key = new Key(uri, conf, unique.getAndIncrement());
      return getInternal(uri, conf, key);
    }

    private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{
     
      FileSystem fs = null;

      // Is there an existing FileSystem being created or in cache?
      synchronized (this) {
        // Somebody else is creating the same FileSystem, so wait.
        while (pending.contains(key)) {
          try {
            wait();
          } catch (InterruptedException e) {
          }
        }
        fs = map.get(key);
       
        if (fs != null) {
          return fs;
        }
        // We are creating one, so add the key to pending creates.
        pending.add(key);
      }
     
      try {
        // Create one
        fs = createFileSystem(uri, conf, key);

        // if sampling is enabled, and fs is the specified type of file system,
        // and this instance is randomly chosen to be in the sample set, then
        // wrap fs with a sample FS.
        if (conf.get("fs.sample.impl", null) != null) {
          Class<?> underlying = conf.getClass("fs.sample.underlying", null);
          if (underlying != null && underlying.equals(fs.getClass())) {
            if (inSampleSet(conf)) {
              fs = createSampleFsWrapper(conf, fs);
              fs.key = key;
            }
          }
        }
       
        // Add it back to the cache
        synchronized (this) {
          map.put(key, fs);
          if (conf.getBoolean("dfs.client.shutdownhook.enable", true)) {
            enableShutdownHook(fs);
          }
        }
      } finally {
        // Make sure we remove the pending key even if createFileSystem
        // throws some exceptions.
        synchronized (this) {
          pending.remove(key);
          notifyAll();
        }
      }
      return fs;
    }

    /** Determines whether a given FS session is part of a sample,
     *  in which case a sample FS is used.  Based on the config
     *  options, it is possible for the sample to consist of a
     *  determined subset of machines, or a random subset of the
     *  sessions on a given machine.
     */
    private boolean inSampleSet(Configuration conf) {
      String flagPath = conf.get("fs.sample.machine.flagfile", "");
      float sampleRate = conf.getFloat("fs.sample.session.rate",
                                       (float)1.0);
      boolean sample = true;

      if (!flagPath.equals("")) {
        if (!(new File(flagPath).exists())) {
          // a path for the enable flag file is specified,
          // but the file does not exist
          sample = false;
        }
      }

      if (sampleRate < (float)1.0) {
        float rand = (new Random()).nextFloat();
        if (rand > sampleRate) {
          // this FS was randomly chosen not to be in the sample
          sample = false;
        }
      }

      return sample;
    }

    /** Creates a sample FS layer over an underlying FS.  The sample FS
     *  should be a FilterFileSystem.  Also, the sample FS should
     *  assume the underlying FS is already initialized, and not perform
     *  a second initialization of the underlying FS when the sample FS
     *  is initialized.
     */
    private FileSystem createSampleFsWrapper(Configuration conf,
                                             FileSystem underlying)throws IOException {

      Class<?> clazz = conf.getClass("fs.sample.impl", null);
      if (clazz == null) {
        throw new IOException("No sample SampleFileSystem specified");
      }
      Object newFsObj = ReflectionUtils.newInstance(clazz,
                                                    new Class[] {FileSystem.class},
                                                    new Object[] {underlying});
      FilterFileSystem sampleFs = (FilterFileSystem)newFsObj;
      sampleFs.initialize(underlying.getUri(), conf);
      return sampleFs;
    }

    /**
     * Remove removes the given fs from the cache.
     */
    synchronized void remove(FileSystem fs) {
      Key key = fs.key;
      if (fs == map.get(key)) {
        map.remove(key);
        disableShutdownHook(fs);
      }
    }

    /**
     * Close and unmap all FileSystems selected by the given FSSselect,
     * which is either AllSelect, UGISelect, or ShutdownSelect.
     */
    private void closeAll(FSSelect select) throws IOException {
      List<FileSystem> targetFSList = new ArrayList<FileSystem>();
      //Make a pass over the list and collect the filesystems to close
      //we cannot close inline since close() removes the entry from the Map
      synchronized(this) {
        for (Map.Entry<Key, FileSystem> entry : map.entrySet()) {
          final Key key = entry.getKey();
          final FileSystem fs = entry.getValue();
          if (select.needClose(fs)) {
            targetFSList.add(fs);
          }
        }
      }
      List<IOException> exceptions = new ArrayList<IOException>();
      for (FileSystem fs : targetFSList) {
        try {
          fs.close();
        }
        catch(IOException ioe) {
          exceptions.add(ioe);
        }
      }
      if (!exceptions.isEmpty()) {
        throw MultipleIOException.createIOException(exceptions);
      }
    }

    /**
     * enableShutdownHook turns on the shutdownHook for this FileSystem.
     * Caller must hold cache object lock.
     */
    private static void enableShutdownHook(FileSystem fs) {
      if (fs.shutdownHook) {
        return;
      }
      fs.shutdownHook = true;
      if (shutdownHookCount++ == 0 && !clientFinalizer.isAlive()) {
        Runtime.getRuntime().addShutdownHook(clientFinalizer);
      }
    }

    /**
     * disableShutdownHook turns off the shutdownHook for this FileSystem.
     * Caller must hold cache object lock.
     */
    private static void disableShutdownHook(FileSystem fs) {
      if (!fs.shutdownHook) {
        return;
      }
      fs.shutdownHook = false;
      if (--shutdownHookCount == 0 && !clientFinalizer.isAlive()) {
        Runtime.getRuntime().removeShutdownHook(clientFinalizer);
      }
    }

    /** FileSystem.Cache.Key */
    static class Key {
      final String scheme;
      final String authority;
      final String username;
      final long unique;   // an artificial way to make a key unique

      Key(URI uri, Configuration conf) throws IOException {
        this(uri, conf, 0);
      }

      Key(URI uri, Configuration conf, long unique) throws IOException {
        scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase();
        authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase();
        this.unique = unique;
        UserGroupInformation ugi = null;
        try {
          ugi = UserGroupInformation.getUGI(conf);
        } catch(LoginException e) {
          LOG.warn("uri=" + uri, e);
        }
        username = ugi == null? null: ugi.getUserName();
      }

      /** {@inheritDoc} */
      public int hashCode() {
        return (scheme + authority + username).hashCode() + (int)unique;
      }

      static boolean isEqual(Object a, Object b) {
        return a == b || (a != null && a.equals(b));       
      }

      /** {@inheritDoc} */
      public boolean equals(Object obj) {
        if (obj == this) {
          return true;
        }
        if (obj != null && obj instanceof Key) {
          Key that = (Key)obj;
          return isEqual(this.scheme, that.scheme)
                 && isEqual(this.authority, that.authority)
                 && isEqual(this.username, that.username)
                 && (this.unique == that.unique);
        }
        return false;       
      }

      /** {@inheritDoc} */
      public String toString() {
        return username + "@" + scheme + "://" + authority;       
      }
    }
  }
 
  public static final class Statistics {
    private final String scheme;
    private AtomicLong bytesRead = new AtomicLong();
    private AtomicLong bytesLocalRead = new AtomicLong();
    private AtomicLong bytesRackLocalRead = new AtomicLong();
    private AtomicLong bytesWritten = new AtomicLong();
    private AtomicLong filesCreated = new AtomicLong();
    private AtomicLong filesRead = new AtomicLong();
    private AtomicLong cntWriteException = new AtomicLong();
    private AtomicLong cntReadException = new AtomicLong();
   
    public Statistics(String scheme) {
      this.scheme = scheme;
    }

    /**
     * Increment the bytes read in the statistics
     * @param newBytes the additional bytes read
     */
    public void incrementBytesRead(long newBytes) {
      bytesRead.getAndAdd(newBytes);
    }
   
    /**
     * Increment the bytes read in the statistics if it is from local machine
     *
     * @param newBytes
     *          the additional bytes read
     */
    public void incrementLocalBytesRead(long newBytes) {
      bytesLocalRead.getAndAdd(newBytes);
    }

    /**
     * Increment the bytes read in the statistics if it is considered local from
     * network topology's view
     *
     * @param newBytes
     *          the additional bytes read
     */
    public void incrementRackLocalBytesRead(long newBytes) {
      bytesRackLocalRead.getAndAdd(newBytes);
    }

    /**
     * Increment the bytes written in the statistics
     * @param newBytes the additional bytes written
     */
    public void incrementBytesWritten(long newBytes) {
      bytesWritten.getAndAdd(newBytes);
    }
   
    /**
     * Increment the files created in the statistics
     */
    public void incrementFilesCreated() {
      filesCreated.getAndAdd(1);
    }

    /**
     * Increment the files read in the statistics
     */
    public void incrementFilesRead() {
      filesRead.getAndAdd(1);
    }

    /**
     * Increment the count of read exceptions in the statistics
     */
    public void incrementCntReadException() {
      cntReadException.getAndAdd(1);
    }

    /**
     * Increment the count of write exceptions in the statistics
     */
    public void incrementCntWriteException() {
      cntWriteException.getAndAdd(1);
    }

    /**
     * Get the total number of bytes read
     * @return the number of bytes
     */
    public long getBytesRead() {
      return bytesRead.get();
    }

    /**
     * Get the total number of bytes read from local host
     * @return the number of bytes
     */
    public long getLocalBytesRead() {
      return bytesLocalRead.get();
    }

    /**
     * Get the total number of bytes read if it is local from network topology
     * point of the view.
     *
     * @return the number of bytes
     */
    public long getRackLocalBytesRead() {
      return bytesRackLocalRead.get();
    }
   
    /**
     * Get the total number of bytes written
     * @return the number of bytes
     */
    public long getBytesWritten() {
      return bytesWritten.get();
    }

    /**
     * Get the number of files created
     * @return the number of files
     */
    public long getFilesCreated() {
      return filesCreated.get();
    }

    /**
     * Get the number of files read
     * @return the number of files
     */
    public long getFilesRead() {
      return filesRead.get();
    }

    /**
     * Get the count of read exceptions
     * @return count of exceptions
     */
    public long getCntReadException() {
      return cntReadException.get();
    }

    /**
     * Get the count of write exceptions
     * @return count of exceptions
     */
    public long getCntWriteException() {
      return cntWriteException.get();
    }   
   
    public String toString() {
      return bytesRead + " bytes read, " + bytesLocalRead + " bytes local read, "
          + bytesRackLocalRead + " bytes rack-local read, " + bytesWritten + " bytes written, and "
          + filesCreated + " files created, " + filesRead + " files read, " +
          cntReadException + " read exceptions, " + cntWriteException + " write exceptions";
    }
   
    /**
     * Reset the counts of bytes to 0.
     */
    public void reset() {
      bytesWritten.set(0);
      bytesRead.set(0);
      bytesLocalRead.set(0);
      bytesRackLocalRead.set(0);
      filesCreated.set(0);
      filesRead.set(0);
      cntReadException.set(0);
      cntWriteException.set(0);
    }
   
    /**
     * Get the uri scheme associated with this statistics object.
     * @return the schema associated with this set of statistics
     */
    public String getScheme() {
      return scheme;
    }
  }
 
  /**
   * Get the Map of Statistics object indexed by URI Scheme.
   * @return a Map having a key as URI scheme and value as Statistics object
   * @deprecated use {@link #getAllStatistics} instead
   */
  public static synchronized Map<String, Statistics> getStatistics() {
    Map<String, Statistics> result = new HashMap<String, Statistics>();
    for(Statistics stat: statisticsTable.values()) {
      result.put(stat.getScheme(), stat);
    }
    return result;
  }

  /**
   * Return the FileSystem classes that have Statistics
   */
  public static synchronized List<Statistics> getAllStatistics() {
    return new ArrayList<Statistics>(statisticsTable.values());
  }
 
  /**
   * Get the statistics for a particular file system
   * @param cls the class to lookup
   * @return a statistics object
   */
  public static synchronized
  Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) {
    Statistics result = statisticsTable.get(cls);
    if (result == null) {
      result = new Statistics(scheme);
      statisticsTable.put(cls, result);
    }
    return result;
  }
 
  public static synchronized void clearStatistics() {
    for(Statistics stat: statisticsTable.values()) {
      stat.reset();
    }
  }

  public static synchronized
  void printStatistics() throws IOException {
    for (Map.Entry<Class<? extends FileSystem>, Statistics> pair:
            statisticsTable.entrySet()) {
      System.out.println("  FileSystem " + pair.getKey().getName() +
                         ": " + pair.getValue());
    }
  }


  public static boolean hasGlobComponent(Path p) throws IOException {
    String filename = p.toUri().getPath();
    String [] components = filename.split(Path.SEPARATOR);

    for(String component: components) {
      FileSystem.GlobFilter fp = new FileSystem.GlobFilter(component);
      if (fp.hasPattern())
        return true;
    }

    return false;
  }

  /**
   * FSSelect is an interface for figuring out which FileSystems to
   * close in a closeAll() call.
   * Its method needClose() returns true if this FS should be closed.
   */
  interface FSSelect {
    public boolean needClose(FileSystem fs);
  }

  /**
   * AllSelect is for closing all FileSystems.
   */
  static class AllSelect implements FSSelect {
    public boolean needClose(FileSystem fs) {
      return fs != null;
    }
  }

  /**
   * ShutdownSelect is for closing FileSystems configured with
   * shutdownHook.
   */
  static class ShutdownSelect implements FSSelect {
    public boolean needClose(FileSystem fs) {
        return fs != null && fs.shutdownHook;
    }
  }

  /**
   * UGISelect is for closing FileSystems that match this
   * UserGroupinformation.
   */
  static class UGISelect implements FSSelect {
    UGISelect(UserGroupInformation ugi) {
      this.uginame = ugi.getUserName();
    }
    public boolean needClose(FileSystem fs) {
      return fs != null && fs.key != null && uginame.equals(fs.key.username);
    }
    private String uginame;
  }
}
TOP

Related Classes of org.apache.hadoop.fs.FileSystem$GlobFilter

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.