Package org.apache.hive.hcatalog.templeton

Source Code of org.apache.hive.hcatalog.templeton.StreamOutputWriter

/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hive.hcatalog.templeton;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Semaphore;

import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.ExecuteException;
import org.apache.commons.exec.ExecuteWatchdog;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.Shell;

class StreamOutputWriter extends Thread
{
  InputStream is;
  String type;
  PrintWriter out;

  StreamOutputWriter(InputStream is, String type, OutputStream outStream)
  {
    this.is = is;
    this.type = type;
    this.out = new PrintWriter(outStream, true);
  }

  @Override
  public void run()
  {
    try
    {
      BufferedReader br =
        new BufferedReader(new InputStreamReader(is));
      String line = null;
      while ( (line = br.readLine()) != null){
        out.println(line);
      }
    } catch (IOException ioe)
    {
      ioe.printStackTrace()
    }
  }
}

/**
* Execute a local program.  This is a singleton service that will
* execute programs as non-privileged users on the local box.  See
* ExecService.run and ExecService.runUnlimited for details.
*/
public class ExecServiceImpl implements ExecService {
  private static final Log LOG = LogFactory.getLog(ExecServiceImpl.class);
  private static AppConfig appConf = Main.getAppConfigInstance();

  private static volatile ExecServiceImpl theSingleton;

  /** Windows CreateProcess synchronization object */
  private static final Object WindowsProcessLaunchLock = new Object();

  /**
   * Retrieve the singleton.
   */
  public static synchronized ExecServiceImpl getInstance() {
    if (theSingleton == null) {
      theSingleton = new ExecServiceImpl();
    }
    return theSingleton;
  }

  private Semaphore avail;

  private ExecServiceImpl() {
    avail = new Semaphore(appConf.getInt(AppConfig.EXEC_MAX_PROCS_NAME, 16));
  }

  /**
   * Run the program synchronously as the given user. We rate limit
   * the number of processes that can simultaneously created for
   * this instance.
   *
   * @param program   The program to run
   * @param args      Arguments to pass to the program
   * @param env       Any extra environment variables to set
   * @return The result of the run.
   */
  public ExecBean run(String program, List<String> args,
            Map<String, String> env)
    throws NotAuthorizedException, BusyException, ExecuteException, IOException {
    boolean aquired = false;
    try {
      aquired = avail.tryAcquire();
      if (aquired) {
        return runUnlimited(program, args, env);
      } else {
        throw new BusyException();
      }
    } finally {
      if (aquired) {
        avail.release();
      }
    }
  }

  /**
   * Run the program synchronously as the given user.  Warning:
   * CommandLine will trim the argument strings.
   *
   * @param program   The program to run.
   * @param args      Arguments to pass to the program
   * @param env       Any extra environment variables to set
   * @return The result of the run.
   */
  public ExecBean runUnlimited(String program, List<String> args,
                 Map<String, String> env)
    throws NotAuthorizedException, ExecuteException, IOException {
    try {
      return auxRun(program, args, env);
    } catch (IOException e) {
      File cwd = new java.io.File(".");
      if (cwd.canRead() && cwd.canWrite())
        throw e;
      else
        throw new IOException("Invalid permissions on Templeton directory: "
          + cwd.getCanonicalPath());
    }
  }

  private ExecBean auxRun(String program, List<String> args, Map<String, String> env)
    throws NotAuthorizedException, ExecuteException, IOException {
    DefaultExecutor executor = new DefaultExecutor();
    executor.setExitValues(null);

    // Setup stdout and stderr
    int nbytes = appConf.getInt(AppConfig.EXEC_MAX_BYTES_NAME, -1);
    ByteArrayOutputStream outStream = new MaxByteArrayOutputStream(nbytes);
    ByteArrayOutputStream errStream = new MaxByteArrayOutputStream(nbytes);
    executor.setStreamHandler(new PumpStreamHandler(outStream, errStream));

    // Only run for N milliseconds
    int timeout = appConf.getInt(AppConfig.EXEC_TIMEOUT_NAME, 0);
    ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout);
    executor.setWatchdog(watchdog);

    CommandLine cmd = makeCommandLine(program, args);

    LOG.info("Running: " + cmd);
    ExecBean res = new ExecBean();

    if(Shell.WINDOWS){
      //The default executor is sometimes causing failure on windows. hcat
      // command sometimes returns non zero exit status with it. It seems
      // to hit some race conditions on windows.
      env = execEnv(env);
      String[] envVals = new String[env.size()];
      int i=0;
      for( Entry<String, String> kv : env.entrySet()){
        envVals[i++] = kv.getKey() + "=" + kv.getValue();
        LOG.info("Setting " +  kv.getKey() + "=" + kv.getValue());
      }

      Process proc;
      synchronized (WindowsProcessLaunchLock) {
        // To workaround the race condition issue with child processes
        // inheriting unintended handles during process launch that can
        // lead to hangs on reading output and error streams, we
        // serialize process creation. More info available at:
        // http://support.microsoft.com/kb/315939
        proc = Runtime.getRuntime().exec(cmd.toStrings(), envVals);
      }

      //consume stderr
      StreamOutputWriter errorGobbler = new
        StreamOutputWriter(proc.getErrorStream(), "ERROR", errStream);

      //consume stdout
      StreamOutputWriter outputGobbler = new
        StreamOutputWriter(proc.getInputStream(), "OUTPUT", outStream);

      //start collecting input streams
      errorGobbler.start();
      outputGobbler.start();
      //execute
      try{
        res.exitcode = proc.waitFor();
      } catch (InterruptedException e) {
        throw new IOException(e);
      }
      //flush
      errorGobbler.out.flush();
      outputGobbler.out.flush();
    }
    else {
      res.exitcode = executor.execute(cmd, execEnv(env));
    }

    String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME);
    res.stdout = outStream.toString(enc);
    res.stderr = errStream.toString(enc);
    try {
      watchdog.checkException();
    }
    catch (Exception ex) {
      LOG.error("Command: " + cmd + " failed:", ex);
    }
    if(watchdog.killedProcess()) {
      String msg = " was terminated due to timeout(" + timeout + "ms).  See " + AppConfig
              .EXEC_TIMEOUT_NAME + " property";
      LOG.warn("Command: " + cmd + msg);
      res.stderr += " Command " + msg;
    }

    return res;
  }

  private CommandLine makeCommandLine(String program,
                    List<String> args)
    throws NotAuthorizedException, IOException {
    String path = validateProgram(program);
    CommandLine cmd = new CommandLine(path);
    if (args != null)
      for (String arg : args)
        cmd.addArgument(arg, false);

    return cmd;
  }

  /**
   * Build the environment used for all exec calls.
   *
   * @return The environment variables.
   */
  public Map<String, String> execEnv(Map<String, String> env) {
    HashMap<String, String> res = new HashMap<String, String>();

    for (String key : appConf.getStrings(AppConfig.EXEC_ENVS_NAME)) {
      String val = System.getenv(key);
      if (val != null) {
        res.put(key, val);
      }
    }
    if (env != null)
      res.putAll(env);
    for (Map.Entry<String, String> envs : res.entrySet()) {
      LOG.info("Env " + envs.getKey() + "=" + envs.getValue());
    }
    return res;
  }

  /**
   * Given a program name, lookup the fully qualified path.  Throws
   * an exception if the program is missing or not authorized.
   *
   * @param path      The path of the program.
   * @return The path of the validated program.
   */
  public String validateProgram(String path)
    throws NotAuthorizedException, IOException {
    File f = new File(path);
    if (f.canExecute()) {
      return f.getCanonicalPath();
    } else {
      throw new NotAuthorizedException("Unable to access program: " + path);
    }
  }
}
TOP

Related Classes of org.apache.hive.hcatalog.templeton.StreamOutputWriter

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.