* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* All classes in this package contain code taken from

* https://github.com/apache/hadoop-common/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?source=cc

* and

* https://github.com/hortonworks/simple-yarn-app

* and

* https://github.com/yahoo/storm-yarn/blob/master/src/main/java/com/yahoo/storm/yarn/StormOnYarn.java

*

* The Flink jar is uploaded to HDFS by this client.

* The application master and all the TaskManager containers get the jar file downloaded

* by YARN into their local fs.

*

*/

  private static final Logger LOG = LoggerFactory.getLogger(Client.class);

  /**

   * Command Line argument options

   */

  private static final Option QUERY = new Option("q","query",false, "Display available YARN resources (memory, cores)");

  // --- or ---

  private static final Option VERBOSE = new Option("v","verbose",false, "Verbose debug mode");

  private static final Option GEN_CONF = new Option("g","generateConf",false, "Place default configuration file in current directory");

  private static final Option QUEUE = new Option("qu","queue",true, "Specify YARN queue.");

  private static final Option SHIP_PATH = new Option("t","ship",true, "Ship files in the specified directory (t for transfer)");

  private static final Option FLINK_CONF_DIR = new Option("c","confDir",true, "Path to Flink configuration directory");

  private static final Option FLINK_JAR = new Option("j","jar",true, "Path to Flink jar file");

  private static final Option JM_MEMORY = new Option("jm","jobManagerMemory",true, "Memory for JobManager Container [in MB]");

  private static final Option TM_MEMORY = new Option("tm","taskManagerMemory",true, "Memory per TaskManager Container [in MB]");

  private static final Option TM_CORES = new Option("tmc","taskManagerCores",true, "Virtual CPU cores per TaskManager");

  private static final Option CONTAINER = new Option("n","container",true, "Number of Yarn container to allocate (=Number of"

      + " Task Managers)");

  private static final Option SLOTS = new Option("s","slots",true, "Number of slots per TaskManager");

  /**

   * Dynamic properties allow the user to specify additional configuration values with -D, such as

   *  -Dfs.overwrite-files=true  -Dtaskmanager.network.numberOfBuffers=16368

   */

  private static final Option DYNAMIC_PROPERTIES = new Option("D", true, "Dynamic properties");

  /**

   * Constants,

   * all starting with ENV_ are used as environment variables to pass values from the Client

   * to the Application Master.

   */

  public final static String ENV_TM_MEMORY = "_CLIENT_TM_MEMORY";

  public final static String ENV_TM_CORES = "_CLIENT_TM_CORES";

  public final static String ENV_TM_COUNT = "_CLIENT_TM_COUNT";

  public final static String ENV_APP_ID = "_APP_ID";

  public final static String ENV_APP_NUMBER = "_APP_NUMBER";

  public final static String FLINK_JAR_PATH = "_FLINK_JAR_PATH"; // the Flink jar resource location (in HDFS).

  public static final String ENV_CLIENT_HOME_DIR = "_CLIENT_HOME_DIR";

  public static final String ENV_CLIENT_SHIP_FILES = "_CLIENT_SHIP_FILES";

  public static final String ENV_CLIENT_USERNAME = "_CLIENT_USERNAME";

  public static final String ENV_AM_PRC_PORT = "_AM_PRC_PORT";

  public static final String ENV_SLOTS = "_SLOTS";

  public static final String ENV_DYNAMIC_PROPERTIES = "_DYNAMIC_PROPERTIES";

  private static final String CONFIG_FILE_NAME = "flink-conf.yaml";

 

  /**

   * Seconds to wait between each status query to the AM.

   */

  private static final int CLIENT_POLLING_INTERVALL = 3;

  /**

   * Minimum memory requirements, checked by the Client.

   */

  private static final int MIN_JM_MEMORY = 128;

  private static final int MIN_TM_MEMORY = 128;

  private Configuration conf;

  private YarnClient yarnClient;

  private ClientMasterControl cmc;

  private File yarnPropertiesFile;

  /**

   * Files (usually in a distributed file system) used for the YARN session of Flink.

   * Contains configuration files and jar files.

   */

  private Path sessionFilesDir;

  /**

   * If the user has specified a different number of slots, we store them here

   */

  private int slots = -1;

 

  public void run(String[] args) throws Exception {

    if(UserGroupInformation.isSecurityEnabled()) {

      throw new RuntimeException("Flink YARN client does not have security support right now."

          + "File a bug, we will fix it asap");

    }

    //Utils.logFilesInCurrentDirectory(LOG);

    //

    //  Command Line Options

    //

    Options options = new Options();

    options.addOption(VERBOSE);

    options.addOption(FLINK_CONF_DIR);

    options.addOption(FLINK_JAR);

    options.addOption(JM_MEMORY);

    options.addOption(TM_MEMORY);

    options.addOption(TM_CORES);

    options.addOption(CONTAINER);

    options.addOption(GEN_CONF);

    options.addOption(QUEUE);

    options.addOption(QUERY);

    options.addOption(SHIP_PATH);

    options.addOption(SLOTS);

    options.addOption(DYNAMIC_PROPERTIES);

    CommandLineParser parser = new PosixParser();

    CommandLine cmd = null;

    try {

      cmd = parser.parse( options, args);

    } catch(MissingOptionException moe) {

      System.out.println(moe.getMessage());

      printUsage();

      System.exit(1);

    }

    // Jar Path

    Path localJarPath;

    if(cmd.hasOption(FLINK_JAR.getOpt())) {

      String userPath = cmd.getOptionValue(FLINK_JAR.getOpt());

      if(!userPath.startsWith("file://")) {

        userPath = "file://" + userPath;

      }

      localJarPath = new Path(userPath);

    } else {

      localJarPath = new Path("file://"+Client.class.getProtectionDomain().getCodeSource().getLocation().getPath());

    }

    if(cmd.hasOption(GEN_CONF.getOpt())) {

      LOG.info("Placing default configuration in current directory");

      File outFile = generateDefaultConf(localJarPath);

      LOG.info("File written to "+outFile.getAbsolutePath());

      System.exit(0);

    }

    // Conf Path

    Path confPath = null;

    String confDirPath = "";

    if(cmd.hasOption(FLINK_CONF_DIR.getOpt())) {

      confDirPath = cmd.getOptionValue(FLINK_CONF_DIR.getOpt())+"/";

      File confFile = new File(confDirPath+CONFIG_FILE_NAME);

      if(!confFile.exists()) {

        LOG.error("Unable to locate configuration file in "+confFile);

        System.exit(1);

      }

      confPath = new Path(confFile.getAbsolutePath());

    } else {

      System.out.println("No configuration file has been specified");

      // no configuration path given.

      // -> see if there is one in the current directory

      File currDir = new File(".");

      File[] candidates = currDir.listFiles(new FilenameFilter() {

        @Override

        public boolean accept(final File dir, final String name) {

          return name != null && name.endsWith(".yaml");

        }

      });

      if(candidates == null || candidates.length == 0) {

        System.out.println("No configuration file has been found in current directory.\n"

            + "Copying default.");

        File outFile = generateDefaultConf(localJarPath);

        confPath = new Path(outFile.toURI());

      } else {

        if(candidates.length > 1) {

          System.out.println("Multiple .yaml configuration files were found in the current directory\n"

              + "Please specify one explicitly");

          System.exit(1);

        } else if(candidates.length == 1) {

          confPath = new Path(candidates[0].toURI());

        }

      }

    }

    List<File> shipFiles = new ArrayList<File>();

    // path to directory to ship

    if(cmd.hasOption(SHIP_PATH.getOpt())) {

      String shipPath = cmd.getOptionValue(SHIP_PATH.getOpt());

      File shipDir = new File(shipPath);

      if(shipDir.isDirectory()) {

        shipFiles = new ArrayList<File>(Arrays.asList(shipDir.listFiles(new FilenameFilter() {

          @Override

          public boolean accept(File dir, String name) {

            return !(name.equals(".") || name.equals("..") );

          }

        })));

      } else {

        LOG.warn("Ship directory is not a directory!");

      }

    }

    boolean hasLogback = false;

    boolean hasLog4j = false;

    //check if there is a logback or log4j file

    if(confDirPath.length() > 0) {

      File logback = new File(confDirPath+"/logback.xml");

      if(logback.exists()) {

        shipFiles.add(logback);

        hasLogback = true;

      }

      File log4j = new File(confDirPath+"/log4j.properties");

      if(log4j.exists()) {

        shipFiles.add(log4j);

        hasLog4j = true;

      }

    }

    // queue

    String queue = "default";

    if(cmd.hasOption(QUEUE.getOpt())) {

      queue = cmd.getOptionValue(QUEUE.getOpt());

    }

    // JobManager Memory

    int jmMemory = 512;

    if(cmd.hasOption(JM_MEMORY.getOpt())) {

      jmMemory = Integer.valueOf(cmd.getOptionValue(JM_MEMORY.getOpt()));

    }

    if(jmMemory < MIN_JM_MEMORY) {

      System.out.println("The JobManager memory is below the minimum required memory amount "

          + "of "+MIN_JM_MEMORY+" MB");

      System.exit(1);

    }

    // Task Managers memory

    int tmMemory = 1024;

    if(cmd.hasOption(TM_MEMORY.getOpt())) {

      tmMemory = Integer.valueOf(cmd.getOptionValue(TM_MEMORY.getOpt()));

    }

    if(tmMemory < MIN_TM_MEMORY) {

      System.out.println("The TaskManager memory is below the minimum required memory amount "

          + "of "+MIN_TM_MEMORY+" MB");

      System.exit(1);

    }

   

    if(cmd.hasOption(SLOTS.getOpt())) {

      slots = Integer.valueOf(cmd.getOptionValue(SLOTS.getOpt()));

    }

   

    String[] dynamicProperties = null;

    if(cmd.hasOption(DYNAMIC_PROPERTIES.getOpt())) {

      dynamicProperties = cmd.getOptionValues(DYNAMIC_PROPERTIES.getOpt());

    }

    String dynamicPropertiesEncoded = StringUtils.join(dynamicProperties, CliFrontend.YARN_DYNAMIC_PROPERTIES_SEPARATOR);

    // Task Managers vcores

    int tmCores = 1;

    if(cmd.hasOption(TM_CORES.getOpt())) {

      tmCores = Integer.valueOf(cmd.getOptionValue(TM_CORES.getOpt()));

    }

    Utils.getFlinkConfiguration(confPath.toUri().getPath());

    int jmPort = GlobalConfiguration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 0);

    if(jmPort == 0) {

      LOG.warn("Unable to find job manager port in configuration!");

      jmPort = ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT;

    }

   

    conf = Utils.initializeYarnConfiguration();

    // intialize HDFS

    LOG.info("Copy App Master jar from local filesystem and add to local environment");

    // Copy the application master jar to the filesystem

    // Create a local resource to point to the destination jar path

    final FileSystem fs = FileSystem.get(conf);

    if(fs.getScheme().startsWith("file")) {

      LOG.warn("The file system scheme is '" + fs.getScheme() + "'. This indicates that the "

          + "specified Hadoop configuration path is wrong and the sytem is using the default Hadoop configuration values."

          + "The Flink YARN client needs to store its files in a distributed file system");

    }

    // Create yarnClient

    yarnClient = YarnClient.createYarnClient();

    yarnClient.init(conf);

    yarnClient.start();

    // Query cluster for metrics

    if(cmd.hasOption(QUERY.getOpt())) {

      showClusterMetrics(yarnClient);

    }

    if(!cmd.hasOption(CONTAINER.getOpt())) {

      LOG.error("Missing required argument "+CONTAINER.getOpt());

      printUsage();

      yarnClient.stop();

      System.exit(1);

    }

    // TM Count

    final int taskManagerCount = Integer.valueOf(cmd.getOptionValue(CONTAINER.getOpt()));

    System.out.println("Using values:");

    System.out.println("\tContainer Count = "+taskManagerCount);

    System.out.println("\tJar Path = "+localJarPath.toUri().getPath());

    System.out.println("\tConfiguration file = "+confPath.toUri().getPath());

    System.out.println("\tJobManager memory = "+jmMemory);

    System.out.println("\tTaskManager memory = "+tmMemory);

    System.out.println("\tTaskManager cores = "+tmCores);

    // Create application via yarnClient

    YarnClientApplication app = yarnClient.createApplication();

    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();

    Resource maxRes = appResponse.getMaximumResourceCapability();

    if(tmMemory > maxRes.getMemory() || tmCores > maxRes.getVirtualCores()) {

      LOG.error("The cluster does not have the requested resources for the TaskManagers available!\n"

          + "Maximum Memory: "+maxRes.getMemory() +", Maximum Cores: "+tmCores);

      yarnClient.stop();

      System.exit(1);

    }

    if(jmMemory > maxRes.getMemory() ) {

      LOG.error("The cluster does not have the requested resources for the JobManager available!\n"

          + "Maximum Memory: "+maxRes.getMemory());

      yarnClient.stop();

      System.exit(1);

    }

    int totalMemoryRequired = jmMemory + tmMemory * taskManagerCount;

    ClusterResourceDescription freeClusterMem = getCurrentFreeClusterResources(yarnClient);

    if(freeClusterMem.totalFreeMemory < totalMemoryRequired) {

      LOG.error("This YARN session requires "+totalMemoryRequired+"MB of memory in the cluster. "

          + "There are currently only "+freeClusterMem.totalFreeMemory+"MB available.");

      yarnClient.stop();

      System.exit(1);

    }

    if( tmMemory > freeClusterMem.containerLimit) {

      LOG.error("The requested amount of memory for the TaskManagers ("+tmMemory+"MB) is more than "

          + "the largest possible YARN container: "+freeClusterMem.containerLimit);

      yarnClient.stop();

      System.exit(1);

    }

    if( jmMemory > freeClusterMem.containerLimit) {

      LOG.error("The requested amount of memory for the JobManager ("+jmMemory+"MB) is more than "

          + "the largest possible YARN container: "+freeClusterMem.containerLimit);

      yarnClient.stop();

      System.exit(1);

    }

    // respect custom JVM options in the YAML file

    final String javaOpts = GlobalConfiguration.getString(ConfigConstants.FLINK_JVM_OPTIONS, "");

    // Set up the container launch context for the application master

    ContainerLaunchContext amContainer = Records

        .newRecord(ContainerLaunchContext.class);

    String amCommand = "$JAVA_HOME/bin/java"

          + " -Xmx"+Utils.calculateHeapSize(jmMemory)+"M " +javaOpts;

    if(hasLogback || hasLog4j) {

      amCommand += " -Dlog.file=\""+ApplicationConstants.LOG_DIR_EXPANSION_VAR +"/jobmanager-main.log\"";

    }

    if(hasLogback) {

      amCommand += " -Dlogback.configurationFile=file:logback.xml";

    }

    if(hasLog4j) {

      amCommand += " -Dlog4j.configuration=file:log4j.properties";

    }

   

    amCommand   += " "+ApplicationMaster.class.getName()+" "

          + " 1>"

          + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager-stdout.log"

          + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager-stderr.log";

    amContainer.setCommands(Collections.singletonList(amCommand));

    System.err.println("amCommand="+amCommand);

    // Set-up ApplicationSubmissionContext for the application

    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();

    final ApplicationId appId = appContext.getApplicationId();

    /**

     * All network ports are offsetted by the application number

     * to avoid version port clashes when running multiple Flink sessions

     * in parallel

     */

    int appNumber = appId.getId();

    jmPort = Utils.offsetPort(jmPort, appNumber);

       

    // Setup jar for ApplicationMaster

    LocalResource appMasterJar = Records.newRecord(LocalResource.class);

    LocalResource flinkConf = Records.newRecord(LocalResource.class);

    Path remotePathJar = Utils.setupLocalResource(conf, fs, appId.toString(), localJarPath, appMasterJar, fs.getHomeDirectory());

    Path remotePathConf = Utils.setupLocalResource(conf, fs, appId.toString(), confPath, flinkConf, fs.getHomeDirectory());

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(2);

    localResources.put("flink.jar", appMasterJar);

    localResources.put("flink-conf.yaml", flinkConf);

    // setup security tokens (code from apache storm)

    final Path[] paths = new Path[3 + shipFiles.size()];

    StringBuffer envShipFileList = new StringBuffer();

    // upload ship files

    for (int i = 0; i < shipFiles.size(); i++) {

      File shipFile = shipFiles.get(i);

      LocalResource shipResources = Records.newRecord(LocalResource.class);

      Path shipLocalPath = new Path("file://" + shipFile.getAbsolutePath());

      paths[3 + i] = Utils.setupLocalResource(conf, fs, appId.toString(),

          shipLocalPath, shipResources, fs.getHomeDirectory());

      localResources.put(shipFile.getName(), shipResources);

      envShipFileList.append(paths[3 + i]);

      if(i+1 < shipFiles.size()) {

        envShipFileList.append(',');

      }

    }

    paths[0] = remotePathJar;

    paths[1] = remotePathConf;

    sessionFilesDir = new Path(fs.getHomeDirectory(), ".flink/" + appId.toString() + "/");

    FsPermission permission = new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL);

    fs.setPermission(sessionFilesDir, permission); // set permission for path.

    Utils.setTokensFor(amContainer, paths, this.conf);

    amContainer.setLocalResources(localResources);

    fs.close();

    int amRPCPort = GlobalConfiguration.getInteger(ConfigConstants.YARN_AM_PRC_PORT, ConfigConstants.DEFAULT_YARN_AM_RPC_PORT);

    amRPCPort = Utils.offsetPort(amRPCPort, appNumber);

    // Setup CLASSPATH for ApplicationMaster

    Map<String, String> appMasterEnv = new HashMap<String, String>();

    Utils.setupEnv(conf, appMasterEnv);

    // set configuration values

    appMasterEnv.put(Client.ENV_TM_COUNT, String.valueOf(taskManagerCount));

    appMasterEnv.put(Client.ENV_TM_CORES, String.valueOf(tmCores));

    appMasterEnv.put(Client.ENV_TM_MEMORY, String.valueOf(tmMemory));

    appMasterEnv.put(Client.FLINK_JAR_PATH, remotePathJar.toString() );

    appMasterEnv.put(Client.ENV_APP_ID, appId.toString());

    appMasterEnv.put(Client.ENV_CLIENT_HOME_DIR, fs.getHomeDirectory().toString());

    appMasterEnv.put(Client.ENV_CLIENT_SHIP_FILES, envShipFileList.toString() );

    appMasterEnv.put(Client.ENV_CLIENT_USERNAME, UserGroupInformation.getCurrentUser().getShortUserName());

    appMasterEnv.put(Client.ENV_AM_PRC_PORT, String.valueOf(amRPCPort));

    appMasterEnv.put(Client.ENV_SLOTS, String.valueOf(slots));

    appMasterEnv.put(Client.ENV_APP_NUMBER, String.valueOf(appNumber));

    if(dynamicPropertiesEncoded != null) {

      appMasterEnv.put(Client.ENV_DYNAMIC_PROPERTIES, dynamicPropertiesEncoded);

    }

    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster

    Resource capability = Records.newRecord(Resource.class);

    capability.setMemory(jmMemory);

    capability.setVirtualCores(1);

    appContext.setApplicationName("Flink"); // application name

    appContext.setAMContainerSpec(amContainer);

    appContext.setResource(capability);

    appContext.setQueue(queue);

    // file that we write into the conf/ dir containing the jobManager address and the dop.

    yarnPropertiesFile = new File(confDirPath + CliFrontend.YARN_PROPERTIES_FILE);

    LOG.info("Submitting application master " + appId);

    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);

    YarnApplicationState appState = appReport.getYarnApplicationState();

    boolean told = false;

    char[] el = { '/', '|', '\\', '-'};

    int i = 0;

    int numTaskmanagers = 0;

    int numMessages = 0;

    BufferedReader in = new BufferedReader(new InputStreamReader(System.in));

    while (appState != YarnApplicationState.FINISHED

        && appState != YarnApplicationState.KILLED

        && appState != YarnApplicationState.FAILED) {

      if(!told && appState ==  YarnApplicationState.RUNNING) {

        System.err.println("Flink JobManager is now running on "+appReport.getHost()+":"+jmPort);

        System.err.println("JobManager Web Interface: "+appReport.getTrackingUrl());

        // write jobmanager connect information

        Properties yarnProps = new Properties();

        yarnProps.setProperty(CliFrontend.YARN_PROPERTIES_JOBMANAGER_KEY, appReport.getHost()+":"+jmPort);

        if(slots != -1) {

          yarnProps.setProperty(CliFrontend.YARN_PROPERTIES_DOP, Integer.toString(slots * taskManagerCount) );

        }

        // add dynamic properties

        if(dynamicProperties != null) {

          yarnProps.setProperty(CliFrontend.YARN_PROPERTIES_DYNAMIC_PROPERTIES_STRING, dynamicPropertiesEncoded);

        }

        OutputStream out = new FileOutputStream(yarnPropertiesFile);

        yarnProps.store(out, "Generated YARN properties file");

        out.close();

        yarnPropertiesFile.setReadable(true, false); // readable for all.

        // connect RPC service

        cmc = new ClientMasterControl(new InetSocketAddress(appReport.getHost(), amRPCPort));

        cmc.start();

        Runtime.getRuntime().addShutdownHook(new ClientShutdownHook());

        told = true;

      }

      if(!told) {

        System.err.print(el[i++]+"\r");

        if(i == el.length) {

          i = 0;

        }

        Thread.sleep(500); // wait for the application to switch to RUNNING

      } else {

        int newTmCount = cmc.getNumberOfTaskManagers();

        if(numTaskmanagers != newTmCount) {

          System.err.println("Number of connected TaskManagers changed to "+newTmCount+". "

              + "Slots available: "+cmc.getNumberOfAvailableSlots());

          numTaskmanagers = newTmCount;

        }

        // we also need to show new messages.

        if(cmc.getFailedStatus()) {

          System.err.println("The Application Master failed!\nMessages:\n");

          for(Message m: cmc.getMessages() ) {

            System.err.println("Message: "+m.getMessage());

          }

          System.err.println("Requesting Application Master shutdown");

          cmc.shutdownAM();

          cmc.close();

          System.err.println("Application Master closed.");

        }

        if(cmc.getMessages().size() != numMessages) {

          System.err.println("Received new message(s) from the Application Master");

          List<Message> msg = cmc.getMessages();

          while(msg.size() > numMessages) {

            System.err.println("Message: "+msg.get(numMessages).getMessage());

            numMessages++;

          }

        }

        // wait until CLIENT_POLLING_INTERVALL is over or the user entered something.

        long startTime = System.currentTimeMillis();

        while ((System.currentTimeMillis() - startTime) < CLIENT_POLLING_INTERVALL * 1000

            && !in.ready()) {

          Thread.sleep(200);

        }

        if (in.ready()) {

          String command = in.readLine();

          evalCommand(command);

        }

      }

      appReport = yarnClient.getApplicationReport(appId);

      appState = appReport.getYarnApplicationState();

    }

    LOG.info("Application " + appId + " finished with"

        + " state " + appState + " and "

        + "final state " + appReport.getFinalApplicationStatus() + " at " + appReport.getFinishTime());

    if(appState == YarnApplicationState.FAILED || appState == YarnApplicationState.KILLED ) {

      LOG.warn("Application failed. Diagnostics "+appReport.getDiagnostics());

      LOG.warn("If log aggregation is activated in the Hadoop cluster, we recommend to retreive "

          + "the full application log using this command:\n"

          + "\tyarn logs -applicationId "+appReport.getApplicationId()+"\n"

          + "(It sometimes takes a few seconds until the logs are aggregated)");

    }

  }

  private void printHelp() {

    System.err.println("Available commands:\n"

        + "\t stop : Stop the YARN session\n"

        + "\t allmsg : Show all messages\n");

  }

  private void evalCommand(String command) {

    if(command.equals("help")) {

      printHelp();

    } else if(command.equals("stop") || command.equals("quit") || command.equals("exit")) {

      stopSession();

      System.exit(0);

    } else if(command.equals("allmsg")) {

      System.err.println("All messages from the ApplicationMaster:");

      for(Message m: cmc.getMessages() ) {

        System.err.println("Message: "+m.getMessage());

      }

    } else if(command.startsWith("add")) {

      System.err.println("This feature is not implemented yet!");

//      int n = Integer.valueOf(nStr);

//      System.err.println("Adding "+n+" TaskManagers to the session");

//      cmc.addTaskManagers(n);

    } else {

      System.err.println("Unknown command '"+command+"'");

      printHelp();

    }

  }

  private void cleanUp() throws IOException {

    LOG.info("Deleting files in "+sessionFilesDir );

    FileSystem shutFS = FileSystem.get(conf);

    shutFS.delete(sessionFilesDir, true); // delete conf and jar file.

    shutFS.close();

  }

 

  private void stopSession() {

    try {

      LOG.info("Sending shutdown request to the Application Master");

      cmc.shutdownAM();

      cleanUp();

      cmc.close();

    } catch (Exception e) {

      LOG.warn("Exception while killing the YARN application", e);

    }

    try {

      yarnPropertiesFile.delete();

    } catch (Exception e) {

      LOG.warn("Exception while deleting the JobManager address file", e);

    }

    LOG.info("YARN Client is shutting down");

    yarnClient.stop();

  }

  public class ClientShutdownHook extends Thread {

    @Override

    public void run() {

      stopSession();

    }

  }

  private static class ClusterResourceDescription {

    public int totalFreeMemory;

    public int containerLimit;

  }

  private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException {

    ClusterResourceDescription crd = new ClusterResourceDescription();

    crd.totalFreeMemory = 0;

    crd.containerLimit = 0;

    List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);

    for(NodeReport rep : nodes) {

      int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0 );

      crd.totalFreeMemory += free;

      if(free > crd.containerLimit) {

        crd.containerLimit = free;

      }

    }

    return crd;

  }

  private void printUsage() {

    System.out.println("Usage:");

    HelpFormatter formatter = new HelpFormatter();

    formatter.setWidth(200);

    formatter.setLeftPadding(5);

    formatter.setSyntaxPrefix("   Required");

    Options req = new Options();

    req.addOption(CONTAINER);

    formatter.printHelp(" ", req);

    formatter.setSyntaxPrefix("   Optional");

    Options opt = new Options();

    opt.addOption(VERBOSE);

    opt.addOption(JM_MEMORY);

    opt.addOption(TM_MEMORY);

    opt.addOption(TM_CORES);

    opt.addOption(QUERY);

    opt.addOption(QUEUE);

    opt.addOption(SLOTS);

    opt.addOption(DYNAMIC_PROPERTIES);

    formatter.printHelp(" ", opt);

  }

  private void showClusterMetrics(YarnClient yarnClient)

      throws YarnException, IOException {

    YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

    System.out.println("NodeManagers in the Cluster " + metrics.getNumNodeManagers());

    List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);

    final String format = "|%-16s |%-16s %n";

    System.out.printf("|Property         |Value          %n");

    System.out.println("+---------------------------------------+");

    int totalMemory = 0;

    int totalCores = 0;

    for(NodeReport rep : nodes) {

      final Resource res = rep.getCapability();

      totalMemory += res.getMemory();

      totalCores += res.getVirtualCores();

      System.out.format(format, "NodeID", rep.getNodeId());

      System.out.format(format, "Memory", res.getMemory()+" MB");

      System.out.format(format, "vCores", res.getVirtualCores());

      System.out.format(format, "HealthReport", rep.getHealthReport());

      System.out.format(format, "Containers", rep.getNumContainers());

      System.out.println("+---------------------------------------+");

    }

    System.out.println("Summary: totalMemory "+totalMemory+" totalCores "+totalCores);

    List<QueueInfo> qInfo = yarnClient.getAllQueues();

    for(QueueInfo q : qInfo) {

      System.out.println("Queue: "+q.getQueueName()+", Current Capacity: "+q.getCurrentCapacity()+" Max Capacity: "+q.getMaximumCapacity()+" Applications: "+q.getApplications().size());

    }

    yarnClient.stop();

    System.exit(0);

  }

  private File generateDefaultConf(Path localJarPath) throws IOException,

      FileNotFoundException {

    JarFile jar = null;

    try {

      jar = new JarFile(localJarPath.toUri().getPath());

    } catch(FileNotFoundException fne) {

      LOG.error("Unable to access jar file. Specify jar file or configuration file.", fne);

      System.exit(1);

    }

    InputStream confStream = jar.getInputStream(jar.getEntry("flink-conf.yaml"));

    if(confStream == null) {

      LOG.warn("Given jar file does not contain yaml conf.");

      confStream = this.getClass().getResourceAsStream("flink-conf.yaml");

      if(confStream == null) {

        throw new RuntimeException("Unable to find flink-conf in jar file");

      }

    }

    File outFile = new File("flink-conf.yaml");

    if(outFile.exists()) {

      throw new RuntimeException("File unexpectedly exists");

    }

    FileOutputStream outputStream = new FileOutputStream(outFile);

    int read = 0;

    byte[] bytes = new byte[1024];

    while ((read = confStream.read(bytes)) != -1) {

      outputStream.write(bytes, 0, read);

    }

    confStream.close(); outputStream.close(); jar.close();

    return outFile;

  }

  public static void main(String[] args) throws Exception {

    Client c = new Client();

    c.run(args);

  }

}