Examples of PackagedProgram


Examples of eu.stratosphere.client.program.PackagedProgram

      testMiniCluster.start();
     
      String jarFile = JAR_FILE;
      String testData = getClass().getResource(TEST_DATA_FILE).toString();
     
      PackagedProgram program = new PackagedProgram(new File(jarFile), new String[] { testData });
           
      Client c = new Client(new InetSocketAddress("localhost", TEST_JM_PORT), new Configuration());
      c.run(program, 4, true);
    }
    catch (Throwable t) {
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

      printHelpForRun();
      return 0;
    }
   
    try {
      PackagedProgram program = buildProgram(line);
      if (program == null) {
        printHelpForRun();
        return 1;
      }
     
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

      return 1;
    }

    // -------- build the packaged program -------------
   
    PackagedProgram program;
    try {
      program = buildProgram(line);
    } catch (Throwable t) {
      return handleError(t);
    }
   
    if (program == null) {
      printHelpForInfo();
      return 1;
    }
   
    int parallelism = -1;
    if (line.hasOption(PARALLELISM_OPTION.getOpt())) {
      String parString = line.getOptionValue(PARALLELISM_OPTION.getOpt());
      try {
        parallelism = Integer.parseInt(parString);
      } catch (NumberFormatException e) {
        System.out.println("The value " + parString + " is invalid for the degree of parallelism.");
        printHelpForRun();
        return 1;
      }
     
      if (parallelism <= 0) {
        System.out.println("Invalid value for the degree-of-parallelism. Parallelism must be greater than zero.");
        printHelpForRun();
        return 1;
      }
    }
   
    try {
      // check for description request
      if (description) {
        String descr = program.getDescription();
       
        if (descr != null) {
          System.out.println("-------------------- Program Description ---------------------");
          System.out.println(descr);
          System.out.println("--------------------------------------------------------------");
        } else {
          System.out.println("No description available for this program.");
        }
      }
     
      // check for json plan request
      if (plan) {
        Client client = getClient(line);
        String jsonPlan = client.getOptimizedPlanAsJson(program, parallelism);
       
        if (jsonPlan != null) {
          System.out.println("----------------------- Execution Plan -----------------------");
          System.out.println(jsonPlan);
          System.out.println("--------------------------------------------------------------");
        } else {
          System.out.println("JSON plan could not be compiled.");
        }
      }
     
      return 0;
    }
    catch (Throwable t) {
      return handleError(t);
    }
    finally {
      program.deleteExtractedLibraries();
    }
  }
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

        line.getOptionValue(CLASS_OPTION.getOpt()) :
        null;
       
    try {
      return entryPointClass == null ?
          new PackagedProgram(jarFile, programArgs) :
          new PackagedProgram(jarFile, entryPointClass, programArgs);
    } catch (ProgramInvocationException e) {
      handleError(e);
      return null;
    }
  }
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

     
      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result instanceof PackagedProgram);
     
      PackagedProgram prog = (PackagedProgram) result;
     
      Assert.assertArrayEquals(new String[] {"some", "program", "arguments"}, prog.getArguments());
      Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail("Program caused an exception: " + e.getMessage());
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

     
      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result instanceof PackagedProgram);
     
      PackagedProgram prog = (PackagedProgram) result;
     
      Assert.assertArrayEquals(new String[] {"some", "program", "arguments"}, prog.getArguments());
      Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail("Program caused an exception: " + e.getMessage());
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

     
      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result instanceof PackagedProgram);
     
      PackagedProgram prog = (PackagedProgram) result;
     
      Assert.assertArrayEquals(new String[] {"some", "program", "arguments"}, prog.getArguments());
      Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail("Program caused an exception: " + e.getMessage());
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

     
      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result instanceof PackagedProgram);
     
      PackagedProgram prog = spy((PackagedProgram) result);

      ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) {
        @Override
        public Class<?> loadClass(String name) throws ClassNotFoundException {
          assertTrue(name.equals("org.apache.hadoop.hive.ql.io.RCFileInputFormat"));
          callme[0] = true;
          return String.class; // Intentionally return the wrong class.
        }
      };
      when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);

      Assert.assertArrayEquals(new String[]{"some", "program"}, prog.getArguments());
      Assert.assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());

      Configuration c = new Configuration();
      c.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "devil");
      Client cli = new Client(c);
     
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

        params.remove(0);
      }

      // create the plan
      String[] options = params.isEmpty() ? new String[0] : (String[]) params.toArray(new String[params.size()]);
      PackagedProgram program;
      OptimizedPlan optPlan;
     
      try {
        if (assemblerClass == null) {
          program = new PackagedProgram(jarFile, options);
        } else {
          program = new PackagedProgram(jarFile, assemblerClass, options);
        }
       
        optPlan = client.getOptimizedPlan(program, -1);
       
        if (optPlan == null) {
          throw new Exception("The optimized plan could not be produced.");
        }
      }
      catch (ProgramInvocationException e) {
        // collect the stack trace
        StringWriter sw = new StringWriter();
        PrintWriter w = new PrintWriter(sw);
       
        if (e.getCause() == null) {
          e.printStackTrace(w);
        } else {
          e.getCause().printStackTrace(w);
        }

        showErrorPage(resp, "An error occurred while invoking the program:<br/><br/>"
          + e.getMessage() + "<br/>"
          + "<br/><br/><pre>" + sw.toString() + "</pre>");
        return;
      }
      catch (CompilerException cex) {
        // collect the stack trace
        StringWriter sw = new StringWriter();
        PrintWriter w = new PrintWriter(sw);
        cex.printStackTrace(w);

        showErrorPage(resp, "An error occurred in the compiler:<br/><br/>"
          + cex.getMessage() + "<br/>"
          + (cex.getCause()!= null?"Caused by: " + cex.getCause().getMessage():"")
          + "<br/><br/><pre>" + sw.toString() + "</pre>");
        return;
      }
      catch (Throwable t) {
        // collect the stack trace
        StringWriter sw = new StringWriter();
        PrintWriter w = new PrintWriter(sw);
        t.printStackTrace(w);

        showErrorPage(resp, "An unexpected error occurred:<br/><br/>" + t.getMessage() + "<br/><br/><pre>"
          + sw.toString() + "</pre>");
        return;
      }

      // redirect according to our options
      if (show) {
        // we have a request to show the plan

        // create a UID for the job
        Long uid = null;
        do {
          uid = Math.abs(this.rand.nextLong());
        } while (this.submittedJobs.containsKey(uid));

        // dump the job to a JSON file
        String planName = uid + ".json";
        File jsonFile = new File(this.planDumpDirectory, planName);
        new PlanJSONDumpGenerator().dumpOptimizerPlanAsJSON(optPlan, jsonFile);

        // submit the job only, if it should not be suspended
        if (!suspend) {
          try {
            this.client.run(program, optPlan, false);
          } catch (Throwable t) {
            LOG.error("Error submitting job to the job-manager.", t);
            showErrorPage(resp, t.getMessage());
            return;
          } finally {
            program.deleteExtractedLibraries();
          }
        } else {
          try {
            this.submittedJobs.put(uid, this.client.getJobGraph(program, optPlan));
          }
          catch (ProgramInvocationException piex) {
            LOG.error("Error creating JobGraph from optimized plan.", piex);
            showErrorPage(resp, piex.getMessage());
            return;
          }
          catch (Throwable t) {
            LOG.error("Error creating JobGraph from optimized plan.", t);
            showErrorPage(resp, t.getMessage());
            return;
          }
        }

        // redirect to the plan display page
        resp.sendRedirect("showPlan?id=" + uid + "&suspended=" + (suspend ? "true" : "false"));
      } else {
        // don't show any plan. directly submit the job and redirect to the
        // nephele runtime monitor
        try {
          client.run(program, -1, false);
        } catch (Exception ex) {
          LOG.error("Error submitting job to the job-manager.", ex);
          // HACK: Is necessary because Message contains whole stack trace
          String errorMessage = ex.getMessage().split("\n")[0];
          showErrorPage(resp, errorMessage);
          return;
        } finally {
          program.deleteExtractedLibraries();
        }
        resp.sendRedirect(START_PAGE_URL);
      }
    } else if (action.equals(ACTION_RUN_SUBMITTED_VALUE)) {
      // --------------- run a job that has been submitted earlier, but was -------------------
View Full Code Here

Examples of eu.stratosphere.client.program.PackagedProgram

      resp.setStatus(HttpServletResponse.SC_BAD_REQUEST);
      return;
    }

    // create the pact plan
    PackagedProgram pactProgram;
    try {
      pactProgram = new PackagedProgram(jarFile, new String[0]);
    }
    catch (Throwable t) {
      LOG.info("Instantiating the PactProgram for '" + jarFile.getName() + "' failed.", t);
      resp.setStatus(HttpServletResponse.SC_BAD_REQUEST);
      resp.getWriter().print(t.getMessage());
      return;
    }
   
    String jsonPlan = null;
    String programDescription = null;
   
    try {
      jsonPlan = pactProgram.getPreviewPlan();
    }
    catch (Throwable t) {
      LOG.error("Failed to create json dump of pact program.", t);
    }
   
    try {
      programDescription = pactProgram.getDescription();
    }
    catch (Throwable t) {
      LOG.error("Failed to create description of pact program.", t);
    }
     
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.