* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* Utilities.

*

*/

  /**

   * The object in the reducer are composed of these top level fields.

   */

  public static String HADOOP_LOCAL_FS = "file:///";

  /**

   * ReduceField.

   *

   */

  public static enum ReduceField {

    KEY, VALUE, ALIAS

  };

  private Utilities() {

    // prevent instantiation

  }

  private static Map<String, MapredWork> gWorkMap = Collections

      .synchronizedMap(new HashMap<String, MapredWork>());

  private static final Log LOG = LogFactory.getLog(Utilities.class.getName());

  public static void clearMapRedWork(Configuration job) {

    try {

      Path planPath = new Path(HiveConf.getVar(job, HiveConf.ConfVars.PLAN));

      FileSystem fs = planPath.getFileSystem(job);

      if (fs.exists(planPath)) {

        try {

          fs.delete(planPath, true);

        } catch (IOException e) {

          e.printStackTrace();

        }

      }

    } catch (Exception e) {

    } finally {

      // where a single process works with multiple plans - we must clear

      // the cache before working with the next plan.

      String jobID = getHiveJobID(job);

      if (jobID != null) {

        gWorkMap.remove(jobID);

      }

    }

  }

  public static MapredWork getMapRedWork(Configuration job) {

    MapredWork gWork = null;

    try {

      String jobID = getHiveJobID(job);

      assert jobID != null;

      gWork = gWorkMap.get(jobID);

      if (gWork == null) {

        String jtConf = HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJT);

        String path;

        if (jtConf.equals("local")) {

          String planPath = HiveConf.getVar(job, HiveConf.ConfVars.PLAN);

          path = new Path(planPath).toUri().getPath();

        } else {

          path = "HIVE_PLAN" + jobID;

        }

        InputStream in = new FileInputStream(path);

        MapredWork ret = deserializeMapRedWork(in, job);

        gWork = ret;

        gWork.initialize();

        gWorkMap.put(jobID, gWork);

      }

      return (gWork);

    } catch (Exception e) {

      e.printStackTrace();

      throw new RuntimeException(e);

    }

  }

  public static List<String> getFieldSchemaString(List<FieldSchema> fl) {

    if (fl == null) {

      return null;

    }

    ArrayList<String> ret = new ArrayList<String>();

    for (FieldSchema f : fl) {

      ret.add(f.getName() + " " + f.getType()

          + (f.getComment() != null ? (" " + f.getComment()) : ""));

    }

    return ret;

  }

  /**

   * Java 1.5 workaround. From http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=5015403

   */

  public static class EnumDelegate extends DefaultPersistenceDelegate {

    @Override

    protected Expression instantiate(Object oldInstance, Encoder out) {

      return new Expression(Enum.class, "valueOf", new Object[] {oldInstance.getClass(),

          ((Enum<?>) oldInstance).name()});

    }

    @Override

    protected boolean mutatesTo(Object oldInstance, Object newInstance) {

      return oldInstance == newInstance;

    }

  }

  public static class MapDelegate extends DefaultPersistenceDelegate {

    @Override

    protected Expression instantiate(Object oldInstance, Encoder out) {

      Map oldMap = (Map) oldInstance;

      HashMap newMap = new HashMap(oldMap);

      return new Expression(newMap, HashMap.class, "new", new Object[] {});

    }

    @Override

    protected boolean mutatesTo(Object oldInstance, Object newInstance) {

      return false;

    }

    @Override

    protected void initialize(Class<?> type, Object oldInstance, Object newInstance, Encoder out) {

      java.util.Collection oldO = (java.util.Collection) oldInstance;

      java.util.Collection newO = (java.util.Collection) newInstance;

      if (newO.size() != 0) {

        out.writeStatement(new Statement(oldInstance, "clear", new Object[] {}));

      }

      for (Iterator i = oldO.iterator(); i.hasNext();) {

        out.writeStatement(new Statement(oldInstance, "add", new Object[] {i.next()}));

      }

    }

  }

  public static class SetDelegate extends DefaultPersistenceDelegate {

    @Override

    protected Expression instantiate(Object oldInstance, Encoder out) {

      Set oldSet = (Set) oldInstance;

      HashSet newSet = new HashSet(oldSet);

      return new Expression(newSet, HashSet.class, "new", new Object[] {});

    }

    @Override

    protected boolean mutatesTo(Object oldInstance, Object newInstance) {

      return false;

    }

    @Override

    protected void initialize(Class<?> type, Object oldInstance, Object newInstance, Encoder out) {

      java.util.Collection oldO = (java.util.Collection) oldInstance;

      java.util.Collection newO = (java.util.Collection) newInstance;

      if (newO.size() != 0) {

        out.writeStatement(new Statement(oldInstance, "clear", new Object[] {}));

      }

      for (Iterator i = oldO.iterator(); i.hasNext();) {

        out.writeStatement(new Statement(oldInstance, "add", new Object[] {i.next()}));

      }

    }

  }

  public static class ListDelegate extends DefaultPersistenceDelegate {

    @Override

    protected Expression instantiate(Object oldInstance, Encoder out) {

      List oldList = (List) oldInstance;

      ArrayList newList = new ArrayList(oldList);

      return new Expression(newList, ArrayList.class, "new", new Object[] {});

    }

    @Override

    protected boolean mutatesTo(Object oldInstance, Object newInstance) {

      return false;

    }

    @Override

    protected void initialize(Class<?> type, Object oldInstance, Object newInstance, Encoder out) {

      java.util.Collection oldO = (java.util.Collection) oldInstance;

      java.util.Collection newO = (java.util.Collection) newInstance;

      if (newO.size() != 0) {

        out.writeStatement(new Statement(oldInstance, "clear", new Object[] {}));

      }

      for (Iterator i = oldO.iterator(); i.hasNext();) {

        out.writeStatement(new Statement(oldInstance, "add", new Object[] {i.next()}));

      }

    }

  }

  public static void setMapRedWork(Configuration job, MapredWork w, String hiveScratchDir) {

    try {

      // this is the unique job ID, which is kept in JobConf as part of the plan file name

      String jobID = UUID.randomUUID().toString();

      Path planPath = new Path(hiveScratchDir, jobID);

      HiveConf.setVar(job, HiveConf.ConfVars.PLAN, planPath.toUri().toString());

      // use the default file system of the job

      FileSystem fs = planPath.getFileSystem(job);

      FSDataOutputStream out = fs.create(planPath);

      serializeMapRedWork(w, out);

      // Serialize the plan to the default hdfs instance

      // Except for hadoop local mode execution where we should be

      // able to get the plan directly from the cache

      if (!HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJT).equals("local")) {

        // Set up distributed cache

        DistributedCache.createSymlink(job);

        String uriWithLink = planPath.toUri().toString() + "#HIVE_PLAN" + jobID;

        DistributedCache.addCacheFile(new URI(uriWithLink), job);

        // set replication of the plan file to a high number. we use the same

        // replication factor as used by the hadoop jobclient for job.xml etc.

        short replication = (short) job.getInt("mapred.submit.replication", 10);

        fs.setReplication(planPath, replication);

      }

      // Cache the plan in this process

      w.initialize();

      gWorkMap.put(jobID, w);

    } catch (Exception e) {

      e.printStackTrace();

      throw new RuntimeException(e);

    }

  }

  public static String getHiveJobID(Configuration job) {

    String planPath = HiveConf.getVar(job, HiveConf.ConfVars.PLAN);

    if (planPath != null) {

      return (new Path(planPath)).getName();

    }

    return null;

  }

  public static String serializeExpression(ExprNodeDesc expr) {

    ByteArrayOutputStream baos = new ByteArrayOutputStream();

    XMLEncoder encoder = new XMLEncoder(baos);

    try {

      encoder.writeObject(expr);

    } finally {

      encoder.close();

    }

    try {

      return baos.toString("UTF-8");

    } catch (UnsupportedEncodingException ex) {

      throw new RuntimeException("UTF-8 support required", ex);

    }

  }

  public static ExprNodeDesc deserializeExpression(String s, Configuration conf) {

    byte[] bytes;

    try {

      bytes = s.getBytes("UTF-8");

    } catch (UnsupportedEncodingException ex) {

      throw new RuntimeException("UTF-8 support required", ex);

    }

    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);

    XMLDecoder decoder = new XMLDecoder(bais, null, null);

    try {

      ExprNodeDesc expr = (ExprNodeDesc) decoder.readObject();

      return expr;

    } finally {

      decoder.close();

    }

  }

  /**

   * Serialize a single Task.

   */

  public static void serializeTasks(Task<? extends Serializable> t, OutputStream out) {

    XMLEncoder e = null;

    try {

      e = new XMLEncoder(out);

      // workaround for java 1.5

      e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate());

      e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate());

      e.setPersistenceDelegate(Operator.ProgressCounter.class, new EnumDelegate());

      e.writeObject(t);

    } finally {

      if (null != e) {

        e.close();

      }

    }

  }

  public static class CollectionPersistenceDelegate extends DefaultPersistenceDelegate {

    @Override

    protected Expression instantiate(Object oldInstance, Encoder out) {

      return new Expression(oldInstance, oldInstance.getClass(), "new", null);

    }

    @Override

    protected void initialize(Class type, Object oldInstance, Object newInstance, Encoder out) {

      Iterator ite = ((Collection) oldInstance).iterator();

      while (ite.hasNext()) {

        out.writeStatement(new Statement(oldInstance, "add", new Object[] {ite.next()}));

      }

    }

  }

  /**

   * Serialize the whole query plan.

   */

  public static void serializeQueryPlan(QueryPlan plan, OutputStream out) {

    XMLEncoder e = new XMLEncoder(out);

    e.setExceptionListener(new ExceptionListener() {

      public void exceptionThrown(Exception e) {

        LOG.warn(org.apache.hadoop.util.StringUtils.stringifyException(e));

        throw new RuntimeException("Cannot serialize the query plan", e);

      }

    });

    // workaround for java 1.5

    e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate());

    e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate());

    e.setPersistenceDelegate(Operator.ProgressCounter.class, new EnumDelegate());

    e.setPersistenceDelegate(org.datanucleus.sco.backed.Map.class, new MapDelegate());

    e.setPersistenceDelegate(org.datanucleus.sco.backed.List.class, new ListDelegate());

    e.writeObject(plan);

    e.close();

  }

  /**

   * Deserialize the whole query plan.

   */

  public static QueryPlan deserializeQueryPlan(InputStream in, Configuration conf) {

    XMLDecoder d = null;

    try {

      d = new XMLDecoder(in, null, null);

      QueryPlan ret = (QueryPlan) d.readObject();

      return (ret);

    } finally {

      if (null != d) {

        d.close();

      }

    }

  }

  /**

   * Serialize the mapredWork object to an output stream. DO NOT use this to write to standard

   * output since it closes the output stream. DO USE mapredWork.toXML() instead.

   */

  public static void serializeMapRedWork(MapredWork w, OutputStream out) {

    XMLEncoder e = null;

    try {

      e = new XMLEncoder(out);

      // workaround for java 1.5

      e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate());

      e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate());

      e.writeObject(w);

    } finally {

      if (null != e) {

        e.close();

      }

    }

  }

  public static MapredWork deserializeMapRedWork(InputStream in, Configuration conf) {

    XMLDecoder d = null;

    try {

      d = new XMLDecoder(in, null, null);

      MapredWork ret = (MapredWork) d.readObject();

      return (ret);

    } finally {

      if (null != d) {

        d.close();

      }

    }

  }

  /**

   * Serialize the mapredLocalWork object to an output stream. DO NOT use this to write to standard

   * output since it closes the output stream. DO USE mapredWork.toXML() instead.

   */

  public static void serializeMapRedLocalWork(MapredLocalWork w, OutputStream out) {

    XMLEncoder e = null;

    try {

      e = new XMLEncoder(out);

      // workaround for java 1.5

      e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate());

      e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate());

      e.writeObject(w);

    } finally {

      if (null != e) {

        e.close();

      }

    }

  }

  public static MapredLocalWork deserializeMapRedLocalWork(InputStream in, Configuration conf) {

    XMLDecoder d = null;

    try {

      d = new XMLDecoder(in, null, null);

      MapredLocalWork ret = (MapredLocalWork) d.readObject();

      return (ret);

    } finally {

      if (null != d) {

        d.close();

      }

    }

  }

  /**

   * Tuple.

   *

   * @param <T>

   * @param <V>

   */

  public static class Tuple<T, V> {

    private final T one;

    private final V two;

    public Tuple(T one, V two) {

      this.one = one;

      this.two = two;

    }

    public T getOne() {

      return this.one;

    }

    public V getTwo() {

      return this.two;

    }

  }

  public static TableDesc defaultTd;

  static {

    // by default we expect ^A separated strings

    // This tableDesc does not provide column names. We should always use

    // PlanUtils.getDefaultTableDesc(String separatorCode, String columns)

    // or getBinarySortableTableDesc(List<FieldSchema> fieldSchemas) when

    // we know the column names.

    defaultTd = PlanUtils.getDefaultTableDesc("" + Utilities.ctrlaCode);

  }

  public static final int newLineCode = 10;

  public static final int tabCode = 9;

  public static final int ctrlaCode = 1;

  public static final String INDENT = "  ";

  // Note: When DDL supports specifying what string to represent null,

  // we should specify "NULL" to represent null in the temp table, and then

  // we can make the following translation deprecated.

  public static String nullStringStorage = "\\N";

  public static String nullStringOutput = "NULL";

  public static Random randGen = new Random();

  /**

   * Gets the task id if we are running as a Hadoop job. Gets a random number otherwise.

   */

  public static String getTaskId(Configuration hconf) {

    String taskid = (hconf == null) ? null : hconf.get("mapred.task.id");

    if ((taskid == null) || taskid.equals("")) {

      return ("" + Math.abs(randGen.nextInt()));

    } else {

      /*

       * extract the task and attempt id from the hadoop taskid. in version 17 the leading component

       * was 'task_'. thereafter the leading component is 'attempt_'. in 17 - hadoop also seems to

       * have used _map_ and _reduce_ to denote map/reduce task types

       */

      String ret = taskid.replaceAll(".*_[mr]_", "").replaceAll(".*_(map|reduce)_", "");

      return (ret);

    }

  }

  public static HashMap makeMap(Object... olist) {

    HashMap ret = new HashMap();

    for (int i = 0; i < olist.length; i += 2) {

      ret.put(olist[i], olist[i + 1]);

    }

    return (ret);

  }

  public static Properties makeProperties(String... olist) {

    Properties ret = new Properties();

    for (int i = 0; i < olist.length; i += 2) {

      ret.setProperty(olist[i], olist[i + 1]);

    }

    return (ret);

  }

  public static ArrayList makeList(Object... olist) {

    ArrayList ret = new ArrayList();

    for (Object element : olist) {

      ret.add(element);

    }

    return (ret);

  }

  /**

   * StreamPrinter.

   *

   */

  public static class StreamPrinter extends Thread {

    InputStream is;

    String type;

    PrintStream os;

    public StreamPrinter(InputStream is, String type, PrintStream os) {

      this.is = is;

      this.type = type;

      this.os = os;

    }

    @Override

    public void run() {

      BufferedReader br = null;

      try {

        InputStreamReader isr = new InputStreamReader(is);

        br = new BufferedReader(isr);

        String line = null;

        if (type != null) {

          while ((line = br.readLine()) != null) {

            os.println(type + ">" + line);

          }

        } else {

          while ((line = br.readLine()) != null) {

            os.println(line);

          }

        }

        br.close();

        br=null;

      } catch (IOException ioe) {

        ioe.printStackTrace();

      }finally{

        IOUtils.closeStream(br);

      }

    }

  }

  public static TableDesc getTableDesc(Table tbl) {

    return (new TableDesc(tbl.getDeserializer().getClass(), tbl.getInputFormatClass(), tbl

        .getOutputFormatClass(), tbl.getSchema()));

  }

  // column names and column types are all delimited by comma

  public static TableDesc getTableDesc(String cols, String colTypes) {

    return (new TableDesc(LazySimpleSerDe.class, SequenceFileInputFormat.class,

        HiveSequenceFileOutputFormat.class, Utilities.makeProperties(

        org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode,

        org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS, cols,

        org.apache.hadoop.hive.serde.Constants.LIST_COLUMN_TYPES, colTypes)));

  }

  public static PartitionDesc getPartitionDesc(Partition part) throws HiveException {

    return (new PartitionDesc(part));

  }

  public static PartitionDesc getPartitionDescFromTableDesc(TableDesc tblDesc, Partition part)

      throws HiveException {

    return new PartitionDesc(part, tblDesc);

  }

  public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator<?> work) {

    mr.addMapWork(tbl.getDataLocation().getPath(), alias, work, new PartitionDesc(

        getTableDesc(tbl), (LinkedHashMap<String, String>) null));

  }

  private static String getOpTreeSkel_helper(Operator<?> op, String indent) {

    if (op == null) {

      return "";

    }

    StringBuilder sb = new StringBuilder();

    sb.append(indent);

    sb.append(op.toString());

    sb.append("\n");

    if (op.getChildOperators() != null) {

      for (Object child : op.getChildOperators()) {

        sb.append(getOpTreeSkel_helper((Operator<?>) child, indent + "  "));

      }

    }

    return sb.toString();

  }

  public static String getOpTreeSkel(Operator<?> op) {

    return getOpTreeSkel_helper(op, "");

  }

  private static boolean isWhitespace(int c) {

    if (c == -1) {

      return false;

    }

    return Character.isWhitespace((char) c);

  }

  public static boolean contentsEqual(InputStream is1, InputStream is2, boolean ignoreWhitespace)

      throws IOException {

    try {

      if ((is1 == is2) || (is1 == null && is2 == null)) {

        return true;

      }

      if (is1 == null || is2 == null) {

        return false;

      }

      while (true) {

        int c1 = is1.read();

        while (ignoreWhitespace && isWhitespace(c1)) {

          c1 = is1.read();

        }

        int c2 = is2.read();

        while (ignoreWhitespace && isWhitespace(c2)) {

          c2 = is2.read();

        }

        if (c1 == -1 && c2 == -1) {

          return true;

        }

        if (c1 != c2) {

          break;

        }

      }

    } catch (FileNotFoundException e) {

      e.printStackTrace();

    }

    return false;

  }

  /**

   * convert "From src insert blah blah" to "From src insert ... blah"

   */

  public static String abbreviate(String str, int max) {

    str = str.trim();

    int len = str.length();

    int suffixlength = 20;

    if (len <= max) {

      return str;

    }

    suffixlength = Math.min(suffixlength, (max - 3) / 2);

    String rev = StringUtils.reverse(str);

    // get the last few words

    String suffix = WordUtils.abbreviate(rev, 0, suffixlength, "");

    suffix = StringUtils.reverse(suffix);

    // first few ..

    String prefix = StringUtils.abbreviate(str, max - suffix.length());

    return prefix + suffix;

  }

  public static final String NSTR = "";

  /**

   * StreamStatus.

   *

   */

  public static enum StreamStatus {

    EOF, TERMINATED

  }

  public static StreamStatus readColumn(DataInput in, OutputStream out) throws IOException {

    while (true) {

      int b;

      try {

        b = in.readByte();

      } catch (EOFException e) {

        return StreamStatus.EOF;

      }

      if (b == Utilities.newLineCode) {

        return StreamStatus.TERMINATED;

      }

      out.write(b);

    }

    // Unreachable

  }

  /**

   * Convert an output stream to a compressed output stream based on codecs and compression options

   * specified in the Job Configuration.

   *

   * @param jc

   *          Job Configuration

   * @param out

   *          Output Stream to be converted into compressed output stream

   * @return compressed output stream

   */

  public static OutputStream createCompressedStream(JobConf jc, OutputStream out)

      throws IOException {

    boolean isCompressed = FileOutputFormat.getCompressOutput(jc);

    return createCompressedStream(jc, out, isCompressed);

  }

  /**

   * Convert an output stream to a compressed output stream based on codecs codecs in the Job

   * Configuration. Caller specifies directly whether file is compressed or not

   *

   * @param jc

   *          Job Configuration

   * @param out

   *          Output Stream to be converted into compressed output stream

   * @param isCompressed

   *          whether the output stream needs to be compressed or not

   * @return compressed output stream

   */

  public static OutputStream createCompressedStream(JobConf jc, OutputStream out,

      boolean isCompressed) throws IOException {

    if (isCompressed) {

      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,

          DefaultCodec.class);

      CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);

      return codec.createOutputStream(out);

    } else {

      return (out);