* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* QTestUtil.

*

*/

  private static final Log LOG = LogFactory.getLog("QTestUtil");

  private String testWarehouse;

  private final String testFiles;

  protected final String outDir;

  protected final String logDir;

  private final TreeMap<String, String> qMap;

  private final Set<String> qSkipSet;

  private final Set<String> qSortSet;

  private static final String SORT_SUFFIX = ".sorted";

  public static final HashSet<String> srcTables = new HashSet<String>

    (Arrays.asList(new String [] {

        "src", "src1", "srcbucket", "srcbucket2", "src_json", "src_thrift",

        "src_sequencefile", "srcpart"

      }));

  private ParseDriver pd;

  private Hive db;

  protected HiveConf conf;

  private Driver drv;

  private BaseSemanticAnalyzer sem;

  private FileSystem fs;

  protected final boolean overWrite;

  private CliDriver cliDriver;

  private HadoopShims.MiniMrShim mr = null;

  private HadoopShims.MiniDFSShim dfs = null;

  private boolean miniMr = false;

  private String hadoopVer = null;

  private QTestSetup setup = null;

  public boolean deleteDirectory(File path) {

    if (path.exists()) {

      File[] files = path.listFiles();

      for (File file : files) {

        if (file.isDirectory()) {

          deleteDirectory(file);

        } else {

          file.delete();

        }

      }

    }

    return (path.delete());

  }

  public void copyDirectoryToLocal(Path src, Path dest) throws Exception {

    FileSystem srcFs = src.getFileSystem(conf);

    FileSystem destFs = dest.getFileSystem(conf);

    if (srcFs.exists(src)) {

      FileStatus[] files = srcFs.listStatus(src);

      for (FileStatus file : files) {

        String name = file.getPath().getName();

        Path dfs_path = file.getPath();

        Path local_path = new Path(dest, name);

        // If this is a source table we do not copy it out

        if (srcTables.contains(name)) {

          continue;

        }

        if (file.isDir()) {

          if (!destFs.exists(local_path)) {

            destFs.mkdirs(local_path);

          }

          copyDirectoryToLocal(dfs_path, local_path);

        } else {

          srcFs.copyToLocalFile(dfs_path, local_path);

        }

      }

    }

  }

  static Pattern mapTok = Pattern.compile("(\\.?)(.*)_map_(.*)");

  static Pattern reduceTok = Pattern.compile("(.*)(reduce_[^\\.]*)((\\..*)?)");

  public void normalizeNames(File path) throws Exception {

    if (path.isDirectory()) {

      File[] files = path.listFiles();

      for (File file : files) {

        normalizeNames(file);

      }

    } else {

      Matcher m = reduceTok.matcher(path.getName());

      if (m.matches()) {

        String name = m.group(1) + "reduce" + m.group(3);

        path.renameTo(new File(path.getParent(), name));

      } else {

        m = mapTok.matcher(path.getName());

        if (m.matches()) {

          String name = m.group(1) + "map_" + m.group(3);

          path.renameTo(new File(path.getParent(), name));

        }

      }

    }

  }

  public QTestUtil(String outDir, String logDir) throws Exception {

    this(outDir, logDir, false, "0.20");

  }

  public String getOutputDirectory() {

    return outDir;

  }

  public String getLogDirectory() {

    return logDir;

  }

 

  private String getHadoopMainVersion(String input) {

    if (input == null) {

      return null;

    }

    Pattern p = Pattern.compile("^(\\d+\\.\\d+).*");

    Matcher m = p.matcher(input);

    if (m.matches()) {

      return m.group(1);

    }

    return null;

  }

  public void initConf() throws Exception {

    if (Shell.WINDOWS) {

      convertPathsFromWindowsToHdfs();

    }

    // Plug verifying metastore in for testing.

    conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,

        "org.apache.hadoop.hive.metastore.VerifyingObjectStore");

    if (miniMr) {

      assert dfs != null;

      assert mr != null;

      mr.setupConfiguration(conf);

      // set fs.default.name to the uri of mini-dfs

      String dfsUriString = getHdfsUriString(dfs.getFileSystem().getUri().toString());

      conf.setVar(HiveConf.ConfVars.HADOOPFS, dfsUriString);

      // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs

      conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,

                  (new Path(dfsUriString,

                            "/build/ql/test/data/warehouse/")).toString());

    }

  }

  private void convertPathsFromWindowsToHdfs() {

    // Following local paths are used as HDFS paths in unit tests.

    // It works well in Unix as the path notation in Unix and HDFS is more or less same.

    // But when it comes to Windows, drive letter separator ':' & backslash '\" are invalid

    // characters in HDFS so we need to converts these local paths to HDFS paths before using them

    // in unit tests.

    // hive.exec.scratchdir needs to be set relative to the mini-dfs

    String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);

    conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, getHdfsUriString(orgWarehouseDir));

    String orgTestTempDir = System.getProperty("test.tmp.dir");

    System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir));

    String orgTestDataDir = System.getProperty("test.src.data.dir");

    System.setProperty("test.src.data.dir", getHdfsUriString(orgTestDataDir));

    String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR);

    conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir));

  }

  private String getHdfsUriString(String uriStr) {

    assert uriStr != null;

    if(Shell.WINDOWS) {

      // If the URI conversion is from Windows to HDFS then replace the '\' with '/'

      // and remove the windows single drive letter & colon from absolute path.

      return uriStr.replace('\\', '/')

        .replaceFirst("/[c-zC-Z]:", "/")

        .replaceFirst("^[c-zC-Z]:", "");

    }

    return uriStr;

  }

  public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer)

    throws Exception {

    this.outDir = outDir;

    this.logDir = logDir;

    conf = new HiveConf(Driver.class);

    this.miniMr = miniMr;

    this.hadoopVer = getHadoopMainVersion(hadoopVer);

    qMap = new TreeMap<String, String>();

    qSkipSet = new HashSet<String>();

    qSortSet = new HashSet<String>();

    if (miniMr) {

      dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);

      FileSystem fs = dfs.getFileSystem();

      mr = ShimLoader.getHadoopShims().getMiniMrCluster(conf, 4, getHdfsUriString(fs.getUri().toString()), 1);

    }

    initConf();

    // Use the current directory if it is not specified

    String dataDir = conf.get("test.data.files");

    if (dataDir == null) {

      dataDir = new File(".").getAbsolutePath() + "/data/files";

    }

    testFiles = dataDir;

    String ow = System.getProperty("test.output.overwrite");

    if ((ow != null) && ow.equalsIgnoreCase("true")) {

      overWrite = true;

    } else {

      overWrite = false;

    }

    setup = new QTestSetup();

    setup.preTest(conf);

    init();

  }

  public void shutdown() throws Exception {

    cleanUp();

    setup.tearDown();

    if (mr != null) {

      mr.shutdown();

      mr = null;

    }

    FileSystem.closeAll();

    if (dfs != null) {

      dfs.shutdown();

      dfs = null;

    }

  }

  public void addFile(String qFile) throws Exception {

    File qf = new File(qFile);

    addFile(qf);

  }

  public void addFile(File qf) throws Exception {

    FileInputStream fis = new FileInputStream(qf);

    BufferedInputStream bis = new BufferedInputStream(fis);

    BufferedReader br = new BufferedReader(new InputStreamReader(bis, "UTF8"));

    StringBuilder qsb = new StringBuilder();

    // Read the entire query

    String line;

    while ((line = br.readLine()) != null) {

      qsb.append(line + "\n");

    }

    br.close();

    String query = qsb.toString();

    qMap.put(qf.getName(), query);

    if(checkHadoopVersionExclude(qf.getName(), query)

      || checkOSExclude(qf.getName(), query)) {

      qSkipSet.add(qf.getName());

    }

    if (checkNeedsSort(qf.getName(), query)) {

      qSortSet.add(qf.getName());

    }

  }

  private boolean checkNeedsSort(String fileName, String query) {

    Pattern pattern = Pattern.compile("-- SORT_BEFORE_DIFF");

    Matcher matcher = pattern.matcher(query);

    if (matcher.find()) {

      return true;

    }

    return false;

  }

  private boolean checkHadoopVersionExclude(String fileName, String query){

    // Look for a hint to not run a test on some Hadoop versions

    Pattern pattern = Pattern.compile("-- (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS\\((.*)\\)");

    boolean excludeQuery = false;

    boolean includeQuery = false;

    Set<String> versionSet = new HashSet<String>();

    String hadoopVer = ShimLoader.getMajorVersion();

    Matcher matcher = pattern.matcher(query);

    // Each qfile may include at most one INCLUDE or EXCLUDE directive.

    //

    // If a qfile contains an INCLUDE directive, and hadoopVer does

    // not appear in the list of versions to include, then the qfile

    // is skipped.

    //

    // If a qfile contains an EXCLUDE directive, and hadoopVer is

    // listed in the list of versions to EXCLUDE, then the qfile is

    // skipped.

    //

    // Otherwise, the qfile is included.

    if (matcher.find()) {

      String prefix = matcher.group(1);

      if ("EX".equals(prefix)) {

        excludeQuery = true;

      } else {

        includeQuery = true;

      }

      String versions = matcher.group(2);

      for (String s : versions.split("\\,")) {

        s = s.trim();

        versionSet.add(s);

      }

    }

    if (matcher.find()) {

      //2nd match is not supposed to be there

      String message = "QTestUtil: qfile " + fileName

        + " contains more than one reference to (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS";

      throw new UnsupportedOperationException(message);

    }

    if (excludeQuery && versionSet.contains(hadoopVer)) {

      System.out.println("QTestUtil: " + fileName

        + " EXCLUDE list contains Hadoop Version " + hadoopVer + ". Skipping...");

      return true;

    } else if (includeQuery && !versionSet.contains(hadoopVer)) {

      System.out.println("QTestUtil: " + fileName

        + " INCLUDE list does not contain Hadoop Version " + hadoopVer + ". Skipping...");

      return true;

    }

    return false;

  }

  private boolean checkOSExclude(String fileName, String query){

    // Look for a hint to not run a test on some Hadoop versions

    Pattern pattern = Pattern.compile("-- (EX|IN)CLUDE_OS_WINDOWS");

    // detect whether this query wants to be excluded or included

    // on windows

    Matcher matcher = pattern.matcher(query);

    if (matcher.find()) {

      String prefix = matcher.group(1);

      if ("EX".equals(prefix)) {

        //windows is to be exluded

        if(Shell.WINDOWS){

          System.out.println("Due to the OS being windows " +

                             "adding the  query " + fileName +

                             " to the set of tests to skip");

          return true;

        }

      }

      else  if(!Shell.WINDOWS){

        //non windows to be exluded

        System.out.println("Due to the OS not being windows " +

                           "adding the  query " + fileName +

                           " to the set of tests to skip");

        return true;

      }

    }

    return false;

  }

  /**

   * Clear out any side effects of running tests

   */

  public void clearPostTestEffects () throws Exception {

    setup.postTest(conf);

  }

  /**

   * Clear out any side effects of running tests

   */

  public void clearTestSideEffects () throws Exception {

    // Delete any tables other than the source tables

    // and any databases other than the default database.

    for (String dbName : db.getAllDatabases()) {

      SessionState.get().setCurrentDatabase(dbName);

      for (String tblName : db.getAllTables()) {

        if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) {

          Table tblObj = db.getTable(tblName);

          // dropping index table can not be dropped directly. Dropping the base

          // table will automatically drop all its index table

          if(tblObj.isIndexTable()) {

            continue;

          }

          db.dropTable(dbName, tblName);

        } else {

          // this table is defined in srcTables, drop all indexes on it

         List<Index> indexes = db.getIndexes(dbName, tblName, (short)-1);

          if (indexes != null && indexes.size() > 0) {

            for (Index index : indexes) {

              db.dropIndex(dbName, tblName, index.getIndexName(), true);

            }

          }

        }

      }

      if (!DEFAULT_DATABASE_NAME.equals(dbName)) {

        db.dropDatabase(dbName);

      }

    }

    SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);

    List<String> roleNames = db.getAllRoleNames();

      for (String roleName : roleNames) {

        db.dropRole(roleName);

    }

    // allocate and initialize a new conf since a test can

    // modify conf by using 'set' commands

    conf = new HiveConf (Driver.class);

    initConf();

    db = Hive.get(conf);  // propagate new conf to meta store

    setup.preTest(conf);

  }

  public void cleanUp() throws Exception {

    // Drop any tables that remain due to unsuccessful runs

    for (String s : new String[] {"src", "src1", "src_json", "src_thrift",

        "src_sequencefile", "srcpart", "srcbucket", "srcbucket2", "dest1",

        "dest2", "dest3", "dest4", "dest4_sequencefile", "dest_j1", "dest_j2",

        "dest_g1", "dest_g2", "fetchtask_ioexception"}) {

      db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s);

    }

    // delete any contents in the warehouse dir

    Path p = new Path(testWarehouse);

    FileSystem fs = p.getFileSystem(conf);

    try {

      FileStatus [] ls = fs.listStatus(p);

      for (int i=0; (ls != null) && (i<ls.length); i++) {

        fs.delete(ls[i].getPath(), true);

      }

    } catch (FileNotFoundException e) {

      // Best effort

    }

    FunctionRegistry.unregisterTemporaryUDF("test_udaf");

    FunctionRegistry.unregisterTemporaryUDF("test_error");

  }

  private void runLoadCmd(String loadCmd) throws Exception {

    int ecode = 0;

    ecode = drv.run(loadCmd).getResponseCode();

    drv.close();

    if (ecode != 0) {

      throw new Exception("load command: " + loadCmd

          + " failed with exit code= " + ecode);

    }

    return;

  }

  private void runCreateTableCmd(String createTableCmd) throws Exception {

    int ecode = 0;

    ecode = drv.run(createTableCmd).getResponseCode();

    if (ecode != 0) {

      throw new Exception("create table command: " + createTableCmd

          + " failed with exit code= " + ecode);

    }

    return;

  }

  public void createSources() throws Exception {

    startSessionState();

    conf.setBoolean("hive.test.init.phase", true);

    // Create a bunch of tables with columns key and value

    LinkedList<String> cols = new LinkedList<String>();

    cols.add("key");

    cols.add("value");

    LinkedList<String> part_cols = new LinkedList<String>();

    part_cols.add("ds");

    part_cols.add("hr");

    db.createTable("srcpart", cols, part_cols, TextInputFormat.class,

        IgnoreKeyTextOutputFormat.class);

    Path fpath;

    HashMap<String, String> part_spec = new HashMap<String, String>();

    for (String ds : new String[] {"2008-04-08", "2008-04-09"}) {

      for (String hr : new String[] {"11", "12"}) {

        part_spec.clear();

        part_spec.put("ds", ds);

        part_spec.put("hr", hr);

        // System.out.println("Loading partition with spec: " + part_spec);

        // db.createPartition(srcpart, part_spec);

        fpath = new Path(testFiles, "kv1.txt");

        // db.loadPartition(fpath, srcpart.getName(), part_spec, true);

        runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()

            + "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='"

            + hr + "')");

      }

    }

    ArrayList<String> bucketCols = new ArrayList<String>();

    bucketCols.add("key");

    runCreateTableCmd("CREATE TABLE srcbucket(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE");

    // db.createTable("srcbucket", cols, null, TextInputFormat.class,

    // IgnoreKeyTextOutputFormat.class, 2, bucketCols);

    for (String fname : new String[] {"srcbucket0.txt", "srcbucket1.txt"}) {

      fpath = new Path(testFiles, fname);

      runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()

          + "' INTO TABLE srcbucket");

    }

    runCreateTableCmd("CREATE TABLE srcbucket2(key int, value string) "

        + "CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE");

    // db.createTable("srcbucket", cols, null, TextInputFormat.class,

    // IgnoreKeyTextOutputFormat.class, 2, bucketCols);

    for (String fname : new String[] {"srcbucket20.txt", "srcbucket21.txt",

        "srcbucket22.txt", "srcbucket23.txt"}) {

      fpath = new Path(testFiles, fname);

      runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()

          + "' INTO TABLE srcbucket2");

    }

    for (String tname : new String[] {"src", "src1"}) {

      db.createTable(tname, cols, null, TextInputFormat.class,

          IgnoreKeyTextOutputFormat.class);

    }

    db.createTable("src_sequencefile", cols, null,

        SequenceFileInputFormat.class, SequenceFileOutputFormat.class);

    Table srcThrift =

        new Table(SessionState.get().getCurrentDatabase(), "src_thrift");

    srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName());

    srcThrift.setOutputFormatClass(SequenceFileOutputFormat.class.getName());

    srcThrift.setSerializationLib(ThriftDeserializer.class.getName());

    srcThrift.setSerdeParam(serdeConstants.SERIALIZATION_CLASS, Complex.class

        .getName());

    srcThrift.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT,

        TBinaryProtocol.class.getName());

    db.createTable(srcThrift);

    LinkedList<String> json_cols = new LinkedList<String>();

    json_cols.add("json");

    db.createTable("src_json", json_cols, null, TextInputFormat.class,

        IgnoreKeyTextOutputFormat.class);

    // load the input data into the src table

    fpath = new Path(testFiles, "kv1.txt");

    runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src");

    // load the input data into the src table

    fpath = new Path(testFiles, "kv3.txt");

    runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src1");

    // load the input data into the src_sequencefile table

    fpath = new Path(testFiles, "kv1.seq");

    runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()

        + "' INTO TABLE src_sequencefile");

    // load the input data into the src_thrift table

    fpath = new Path(testFiles, "complex.seq");

    runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()

        + "' INTO TABLE src_thrift");

    // load the json data into the src_json table

    fpath = new Path(testFiles, "json.txt");

    runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()

        + "' INTO TABLE src_json");

    conf.setBoolean("hive.test.init.phase", false);

  }

  public void init() throws Exception {

    // System.out.println(conf.toString());

    testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);

    // conf.logVars(System.out);

    // System.out.flush();

    SessionState.start(conf);

    db = Hive.get(conf);

    fs = FileSystem.get(conf);

    drv = new Driver(conf);

    drv.init();

    pd = new ParseDriver();

    sem = new SemanticAnalyzer(conf);

  }

  public void init(String tname) throws Exception {

    cleanUp();

    createSources();

    LinkedList<String> cols = new LinkedList<String>();

    cols.add("key");

    cols.add("value");

    LinkedList<String> part_cols = new LinkedList<String>();

    part_cols.add("ds");

    part_cols.add("hr");

    db.createTable("dest1", cols, null, TextInputFormat.class,

        IgnoreKeyTextOutputFormat.class);

    db.createTable("dest2", cols, null, TextInputFormat.class,

        IgnoreKeyTextOutputFormat.class);

    db.createTable("dest3", cols, part_cols, TextInputFormat.class,

        IgnoreKeyTextOutputFormat.class);

    Table dest3 = db.getTable("dest3");

    HashMap<String, String> part_spec = new HashMap<String, String>();

    part_spec.put("ds", "2008-04-08");

    part_spec.put("hr", "12");

    db.createPartition(dest3, part_spec);

    db.createTable("dest4", cols, null, TextInputFormat.class,

        IgnoreKeyTextOutputFormat.class);

    db.createTable("dest4_sequencefile", cols, null,

        SequenceFileInputFormat.class, SequenceFileOutputFormat.class);

  }

  public void cliInit(String tname) throws Exception {

    cliInit(tname, true);

  }

  public void cliInit(String tname, boolean recreate) throws Exception {

    if (recreate) {

      cleanUp();

      createSources();

    }

    HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,

    "org.apache.hadoop.hive.ql.security.DummyAuthenticator");

    CliSessionState ss = new CliSessionState(conf);

    assert ss != null;

    ss.in = System.in;

    File qf = new File(outDir, tname);

    File outf = null;

    outf = new File(logDir);

    outf = new File(outf, qf.getName().concat(".out"));

    FileOutputStream fo = new FileOutputStream(outf);

    ss.out = new PrintStream(fo, true, "UTF-8");

    ss.err = new CachingPrintStream(fo, true, "UTF-8");

    ss.setIsSilent(true);

    SessionState oldSs = SessionState.get();

    if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {

      oldSs.out.close();

    }

    SessionState.start(ss);

    cliDriver = new CliDriver();

    if (tname.equals("init_file.q")) {

      ss.initFiles.add("../data/scripts/test_init_file.sql");

    }

    cliDriver.processInitFiles(ss);

  }

  private CliSessionState startSessionState()

      throws FileNotFoundException, UnsupportedEncodingException {

    HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,

        "org.apache.hadoop.hive.ql.security.DummyAuthenticator");

    CliSessionState ss = new CliSessionState(conf);

    assert ss != null;

    SessionState.start(ss);

    return ss;

  }

  public int executeOne(String tname) {

    String q = qMap.get(tname);

    if (q.indexOf(";") == -1) {

      return -1;

    }

    String q1 = q.substring(0, q.indexOf(";") + 1);

    String qrest = q.substring(q.indexOf(";") + 1);

    qMap.put(tname, qrest);

    System.out.println("Executing " + q1);

    return cliDriver.processLine(q1);

  }

  public int execute(String tname) {

    try {

      return drv.run(qMap.get(tname)).getResponseCode();

    } catch (CommandNeedRetryException e) {

      // TODO Auto-generated catch block

      e.printStackTrace();

      return -1;

    }

  }

  public int executeClient(String tname) {

    String commands = qMap.get(tname);

    StringBuilder newCommands = new StringBuilder(commands.length());

    int lastMatchEnd = 0;

    Matcher commentMatcher = Pattern.compile("^--.*$", Pattern.MULTILINE).matcher(commands);

    while (commentMatcher.find()) {

      newCommands.append(commands.substring(lastMatchEnd, commentMatcher.start()));

      newCommands.append(commentMatcher.group().replaceAll("(?<!\\\\);", "\\\\;"));

      lastMatchEnd = commentMatcher.end();

    }

    newCommands.append(commands.substring(lastMatchEnd, commands.length()));

    commands = newCommands.toString();

    return cliDriver.processLine(commands);

  }

  public boolean shouldBeSkipped(String tname) {

    return qSkipSet.contains(tname);

  }

  public void convertSequenceFileToTextFile() throws Exception {

    // Create an instance of hive in order to create the tables

    testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);

    db = Hive.get(conf);

    // Create dest4 to replace dest4_sequencefile

    LinkedList<String> cols = new LinkedList<String>();

    cols.add("key");

    cols.add("value");

    // Move all data from dest4_sequencefile to dest4

    drv

        .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*");

    // Drop dest4_sequencefile

    db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile",

        true, true);

  }

  public int checkNegativeResults(String tname, Exception e) throws Exception {

    File qf = new File(outDir, tname);

    String expf = outPath(outDir.toString(), tname.concat(".out"));

    File outf = null;

    outf = new File(logDir);

    outf = new File(outf, qf.getName().concat(".out"));

    FileWriter outfd = new FileWriter(outf);

    if (e instanceof ParseException) {

      outfd.write("Parse Error: ");

    } else if (e instanceof SemanticException) {

      outfd.write("Semantic Exception: \n");

    } else {

      throw e;

    }

    outfd.write(e.getMessage());

    outfd.close();

    int exitVal = executeDiffCommand(outf.getPath(), expf, false,

                                     qSortSet.contains(qf.getName()));

    if (exitVal != 0 && overWrite) {

      exitVal = overwriteResults(outf.getPath(), expf);

    }

    return exitVal;

  }

  public int checkParseResults(String tname, ASTNode tree) throws Exception {

    if (tree != null) {

      File parseDir = new File(outDir, "parse");

      String expf = outPath(parseDir.toString(), tname.concat(".out"));

      File outf = null;

      outf = new File(logDir);

      outf = new File(outf, tname.concat(".out"));

      FileWriter outfd = new FileWriter(outf);

      outfd.write(tree.toStringTree());

      outfd.close();

      int exitVal = executeDiffCommand(outf.getPath(), expf, false, false);

      if (exitVal != 0 && overWrite) {

        exitVal = overwriteResults(outf.getPath(), expf);

      }

      return exitVal;

    } else {

      throw new Exception("Parse tree is null");

    }

  }

  private final Pattern[] xmlPlanMask = toPattern(new String[] {

      "<java version=\".*\" class=\"java.beans.XMLDecoder\">",

      "<string>.*/tmp/.*</string>",

      "<string>file:.*</string>",

      "<string>pfile:.*</string>",

      "<string>[0-9]{10}</string>",

      "<string>/.*/warehouse/.*</string>"

  });

  public int checkPlan(String tname, List<Task<? extends Serializable>> tasks) throws Exception {

    if (tasks != null) {

      File planDir = new File(outDir, "plan");

      String planFile = outPath(planDir.toString(), tname + ".xml");

      File outf = null;

      outf = new File(logDir);

      outf = new File(outf, tname.concat(".xml"));

      FileOutputStream ofs = new FileOutputStream(outf);

      for (Task<? extends Serializable> plan : tasks) {

        Utilities.serializeObject(plan, ofs);

      }

     

      fixXml4JDK7(outf.getPath());

      maskPatterns(xmlPlanMask, outf.getPath());

      int exitVal = executeDiffCommand(outf.getPath(), planFile, true, false);

      if (exitVal != 0 && overWrite) {

        exitVal = overwriteResults(outf.getPath(), planFile);

      }

      return exitVal;

    } else {

      throw new Exception("Plan is null");

    }

  }

  /**

   * Given the current configurations (e.g., hadoop version and execution mode), return

   * the correct file name to compare with the current test run output.

   * @param outDir The directory where the reference log files are stored.

   * @param testName The test file name (terminated by ".out").

   * @return The file name appended with the configuration values if it exists.

   */

  public String outPath(String outDir, String testName) {

    String ret = (new File(outDir, testName)).getPath();

    // List of configurations. Currently the list consists of hadoop version and execution mode only

    List<String> configs = new ArrayList<String>();

    configs.add(this.hadoopVer);

    Deque<String> stack = new LinkedList<String>();

    StringBuilder sb = new StringBuilder();

    sb.append(testName);

    stack.push(sb.toString());

    // example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17

    for (String s: configs) {

      sb.append('_');

      sb.append(s);

      stack.push(sb.toString());

    }

    while (stack.size() > 0) {

      String fileName = stack.pop();

      File f = new File(outDir, fileName);

      if (f.exists()) {

        ret = f.getPath();

        break;

      }

    }

   return ret;

  }

  /**

   * Fix the XML generated by JDK7 which is slightly different from what's generated by JDK6,

   * causing 40+ test failures. There are mainly two problems:

   *

   * 1. object element's properties, id and class, are in reverse order, i.e.

   *    <object class="org.apache.hadoop.hive.ql.exec.MapRedTask" id="MapRedTask0">

   *    which needs to be fixed to

   *    <object id="MapRedTask0" class="org.apache.hadoop.hive.ql.exec.MapRedTask">

   * 2. JDK introduces Enum as class, i.e.

   *    <object id="GenericUDAFEvaluator$Mode0" class="java.lang.Enum">

   *      <class>org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode</class>

   *    which needs to be fixed to

   *    <object id="GenericUDAFEvaluator$Mode0" class="org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode"

   *     method="valueOf">

   *   

   * Though not elegant, this allows these test cases to pass until we have a better serialization mechanism.

   *

   * Did I mention this is test code?

   *   

   * @param fname the name of the file to fix

   * @throws Exception in case of IO error

   */

  private static void fixXml4JDK7(String fname) throws Exception {

    String version = System.getProperty("java.version");

    if (!version.startsWith("1.7")) {

      return;

    }

   

    BufferedReader in = new BufferedReader(new FileReader(fname));

    BufferedWriter out = new BufferedWriter(new FileWriter(fname + ".orig"));

    String line = null;

    while (null != (line = in.readLine())) {

      out.write(line);

      out.write('\n');

    }

    in.close();

    out.close();

    in = new BufferedReader(new FileReader(fname + ".orig"));

    out = new BufferedWriter(new FileWriter(fname));

   

    while (null != (line = in.readLine())) {

      if (line.indexOf("<object ") == -1 || line.indexOf("class=") == -1) {

        out.write(line);

      } else {

        StringBuilder sb = new StringBuilder();

        String prefix = line.substring(0, line.indexOf("<object") + 7);

        sb.append( prefix );

        String postfix = line.substring(line.lastIndexOf('"') + 1);

        String id = getPropertyValue(line, "id");

        if (id != null)

          sb.append(" id=" + id);

        String cls = getPropertyValue(line, "class");

        assert(cls != null);

        if (cls.equals("\"java.lang.Enum\"")) {

          line = in.readLine();

          cls = "\"" + getElementValue(line, "class") + "\"";

          sb.append(" class=" + cls + " method=\"valueOf\"" );

        } else {

          sb.append(" class=" + cls);

        }

        sb.append(postfix);

        out.write(sb.toString());

      }

     

      out.write('\n');

    }

    in.close();

    out.close();

  }

 

  /**

   * Get the value of a property in line. The returned value has original quotes

   */

  private static String getPropertyValue(String line, String name) {

    int start = line.indexOf( name + "=" );

    if (start == -1)

      return null;

    start += name.length() + 1;

    int end = line.indexOf("\"", start + 1);

    return line.substring( start, end + 1 );

  }

 

  /**

   * Get the value of the element in input. (Note: the returned value has no quotes.)

   */

  private static String getElementValue(String line, String name) {

    assert(line.contains("<" + name + ">"));

    int start = line.indexOf("<" + name + ">") + name.length() + 2;

    int end = line.indexOf("</" + name + ">");

    return line.substring(start, end);

  }

  private Pattern[] toPattern(String[] patternStrs) {

    Pattern[] patterns = new Pattern[patternStrs.length];

    for (int i = 0; i < patternStrs.length; i++) {

      patterns[i] = Pattern.compile(patternStrs[i]);

    }

    return patterns;

  }

  private void maskPatterns(Pattern[] patterns, String fname) throws Exception {

    String maskPattern = "#### A masked pattern was here ####";

    String line;

    BufferedReader in;

    BufferedWriter out;

    in = new BufferedReader(new FileReader(fname));

    out = new BufferedWriter(new FileWriter(fname + ".orig"));

    while (null != (line = in.readLine())) {

      out.write(line);

      out.write('\n');

    }

    in.close();

    out.close();

    in = new BufferedReader(new FileReader(fname + ".orig"));

    out = new BufferedWriter(new FileWriter(fname));

    boolean lastWasMasked = false;

    while (null != (line = in.readLine())) {

      for (Pattern pattern : patterns) {

        line = pattern.matcher(line).replaceAll(maskPattern);

      }

      if (line.equals(maskPattern)) {

        // We're folding multiple masked lines into one.

        if (!lastWasMasked) {

          out.write(line);

          out.write("\n");

          lastWasMasked = true;

        }

      } else {

        out.write(line);

        out.write("\n");

        lastWasMasked = false;

      }

    }

    in.close();

    out.close();

  }

  private final Pattern[] planMask = toPattern(new String[] {

      ".*file:.*",

      ".*pfile:.*",

      ".*hdfs:.*",

      ".*/tmp/.*",

      ".*invalidscheme:.*",

      ".*lastUpdateTime.*",

      ".*lastAccessTime.*",

      ".*lastModifiedTime.*",

      ".*[Oo]wner.*",

      ".*CreateTime.*",

      ".*LastAccessTime.*",

      ".*Location.*",

      ".*LOCATION '.*",

      ".*transient_lastDdlTime.*",

      ".*last_modified_.*",

      ".*at org.*",

      ".*at sun.*",

      ".*at java.*",

      ".*at junit.*",

      ".*Caused by:.*",

      ".*LOCK_QUERYID:.*",

      ".*LOCK_TIME:.*",

      ".*grantTime.*",

      ".*[.][.][.] [0-9]* more.*",

      ".*job_[0-9_]*.*",

      ".*job_local[0-9_]*.*",

      ".*USING 'java -cp.*",

      "^Deleted.*",

  });

  public int checkCliDriverResults(String tname) throws Exception {

    String[] cmdArray;

    assert(qMap.containsKey(tname));

    String outFileName = outPath(outDir, tname + ".out");

    File f = new File(logDir, tname + ".out");

    maskPatterns(planMask, f.getPath());

    int exitVal = executeDiffCommand(f.getPath(),

                                     outFileName, false,

                                     qSortSet.contains(tname));

    if (exitVal != 0 && overWrite) {

      exitVal = overwriteResults(f.getPath(), outFileName);

    }

    return exitVal;

  }

  private static int overwriteResults(String inFileName, String outFileName) throws Exception {

    // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)

    // once Hive uses JAVA 7.

    System.out.println("Overwriting results");

    return executeCmd(new String[] {

        "cp",

        getQuotedString(inFileName),

        getQuotedString(outFileName)

      });

  }

  private static int executeDiffCommand(String inFileName,

      String outFileName,

      boolean ignoreWhiteSpace,

      boolean sortResults

      ) throws Exception {

    int result = 0;

    if (sortResults) {

      // sort will try to open the output file in write mode on windows. We need to

      // close it first.

      SessionState ss = SessionState.get();

      if (ss != null && ss.out != null && ss.out != System.out) {

  ss.out.close();

      }

      String inSorted = inFileName + SORT_SUFFIX;

      String outSorted = outFileName + SORT_SUFFIX;

      result = sortFiles(inFileName, inSorted);

      result |= sortFiles(outFileName, outSorted);

      if (result != 0) {

        System.err.println("ERROR: Could not sort files before comparing");

        return result;

      }