Examples of com.google.appengine.tools.pipeline.JobInfo

Package com.google.appengine.tools.pipeline

Examples of com.google.appengine.tools.pipeline.JobInfo

com.google.appengine.tools.pipeline.JobInfo
A record about a job that has been registered with the framework. A {@code JobInfo} is obtained via the method{@link PipelineService#getJobInfo(String)}. @author rudominer@google.com (Mitch Rudominer)

  private <I, K, V, O, R> void runWithPipeline(MapReduceSettings settings,
      MapReduceSpecification<I, K, V, O, R> mrSpec, Verifier<R> verifier) throws Exception {
    String jobId = pipelineService.startNewPipeline(new MapReduceJob<>(mrSpec, settings));
    assertFalse(jobId.isEmpty());
    executeTasksUntilEmpty("default");
    JobInfo info = pipelineService.getJobInfo(jobId);
    @SuppressWarnings("unchecked")
    MapReduceResult<R> result = (MapReduceResult<R>) info.getOutput();
    assertEquals(JobInfo.State.COMPLETED_SUCCESSFULLY, info.getJobState());
    assertNotNull(result);
    verifier.verify(result);
  }

View Full Code Here

  private <I, O, R> void runWithPipeline(MapSettings settings, MapSpecification<I, O, R> mrSpec,
      Verifier<R> verifier) throws Exception {
    String jobId = pipelineService.startNewPipeline(new MapJob<>(mrSpec, settings));
    assertFalse(jobId.isEmpty());
    executeTasksUntilEmpty("default");
    JobInfo info = pipelineService.getJobInfo(jobId);
    @SuppressWarnings("unchecked")
    MapReduceResult<R> result = (MapReduceResult<R>) info.getOutput();
    assertEquals(JobInfo.State.COMPLETED_SUCCESSFULLY, info.getJobState());
    assertNotNull(result);
    verifier.verify(result);
  }

View Full Code Here

              .setJobName("Shard-retry failed").setKeyMarshaller(Marshallers.getStringMarshaller())
              .setValueMarshaller(Marshallers.getLongMarshaller()).build();
      String jobId = pipelineService.startNewPipeline(new MapReduceJob<>(mrSpec, mrSettings));
      assertFalse(jobId.isEmpty());
      executeTasksUntilEmpty("default");
      JobInfo info = pipelineService.getJobInfo(jobId);
      assertNull(info.getOutput());
      assertEquals(JobInfo.State.STOPPED_BY_ERROR, info.getJobState());
      assertTrue(info.getException().getMessage()
          .matches("Stage map-.* was not completed successfuly \\(status=ERROR, message=.*\\)"));
    }


    // Disallow slice-retry
    runs = new int[][] { {5, 0}, {0, 5}, {4, 1}, {1, 4}};
    for (int[] run : runs) {
      RandomLongInput input = new RandomLongInput(10, shardsCount);
      RougeMapper mapper = new RougeMapper(shardsCount, run[0], run[1]);
      mapper.setAllowSliceRetry(false);
      MapReduceSettings mrSettings = new MapReduceSettings.Builder().setMillisPerSlice(0).build();
      MapReduceSpecification<Long, String, Long, String, String> mrSpec =
          new MapReduceSpecification.Builder<>(input, mapper,
              NoReducer.<String, Long, String>create(), new NoOutput<String, String>())
              .setJobName("Shard-retry failed").setKeyMarshaller(Marshallers.getStringMarshaller())
              .setValueMarshaller(Marshallers.getLongMarshaller()).build();
      String jobId = pipelineService.startNewPipeline(new MapReduceJob<>(mrSpec, mrSettings));
      assertFalse(jobId.isEmpty());
      executeTasksUntilEmpty("default");
      JobInfo info = pipelineService.getJobInfo(jobId);
      assertNull(info.getOutput());
      assertEquals(JobInfo.State.STOPPED_BY_ERROR, info.getJobState());
      assertTrue(info.getException().getMessage()
          .matches("Stage map-.* was not completed successfuly \\(status=ERROR, message=.*\\)"));
    }
  }

View Full Code Here

    MapReduceSettings mrSettings = new MapReduceSettings.Builder().build();
    String jobId = pipelineService.startNewPipeline(
        new MapReduceJob<>(mrSpecBuilder.build(), mrSettings));
    assertFalse(jobId.isEmpty());
    executeTasksUntilEmpty("default");
    JobInfo info = pipelineService.getJobInfo(jobId);
    assertEquals(State.COMPLETED_SUCCESSFULLY, info.getJobState());
    @SuppressWarnings("unchecked")
    MapReduceResult<Boolean> result = (MapReduceResult<Boolean>) info.getOutput();
    assertNotNull(result);
    assertTrue(result.getOutputResult());
  }

View Full Code Here

    preparer.prepare();
    String jobId = pipelineService.startNewPipeline(new MapReduceJob<I, K, V, O, R>(),
        mrSpec, new MapReduceSettings());
    assertFalse(jobId.isEmpty());
    executeTasksUntilEmpty("default");
    JobInfo info = pipelineService.getJobInfo(jobId);
    @SuppressWarnings("unchecked")
    MapReduceResult<R> result = (MapReduceResult) info.getOutput();
    assertNotNull(result);
    verifier.verify(result);
  }

View Full Code Here

        PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
        return getJobInfo(pipelineService, phase, handle);
    }


    protected JobInfo getJobInfo(PipelineService pipelineService, String phase, final String handle) throws Exception {
        JobInfo jobInfo = pipelineService.getJobInfo(handle);
        Assert.assertNotNull("Missing JobInfo - [ " + phase + " ] - handle: " + handle, jobInfo);
        return jobInfo;
    }

View Full Code Here

        return jobInfo;
    }


    protected JobInfo waitToFinish(final String phase, final String handle) throws Exception {
        PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
        JobInfo jobInfo = getJobInfo(pipelineService, phase, handle);
        JobInfo.State state = jobInfo.getJobState();
        int N = 24; // 2min
        while (isRunning(state) && N > 0) {
            N--;
            sync(5 * 1000L); // 5sec
            // new info lookup
            jobInfo = getJobInfo(pipelineService, phase, handle);
            state = jobInfo.getJobState();
        }
        if (N == 0 && isRunning(state)) {
            throw new IllegalStateException("Failed to finish the job [ " + phase + " ]: " + handle + ", info: " + toInfo(jobInfo));
        }
        if (state != JobInfo.State.COMPLETED_SUCCESSFULLY) {

View Full Code Here

        builder.setReducer(NoReducer.create());
        builder.setOutput(new NoOutput());


        final String createHandle = MapReduceJob.start(builder.build(), getSettings());


        JobInfo createJI = waitToFinish("CREATE", createHandle);
        Object create = createJI.getOutput();
        log.warning("----- Create: " + create);


        int mapShardCount = 1;


        builder = new MapReduceSpecification.Builder();
        builder.setJobName("MapReduceTest stats");
        builder.setInput(new DatastoreInput("MapReduceTest", mapShardCount));
        builder.setMapper(new CountMapper());
        builder.setKeyMarshaller(Marshallers.getStringMarshaller());
        builder.setValueMarshaller(Marshallers.getLongMarshaller());
        builder.setReducer(new CountReducer());
        builder.setOutput(new InMemoryOutput<KeyValue<String, Long>>());


        final String countHandle = MapReduceJob.start(builder.build(), getSettings());


        JobInfo countJI = waitToFinish("COUNT", countHandle);
        Object count = countJI.getOutput();
        log.warning("----- Count: " + count);


        Assert.assertTrue(count instanceof MapReduceResult);
        MapReduceResult result = MapReduceResult.class.cast(count);
        int[] chars = toChars(payloads);

View Full Code Here

TOP

Related Classes of com.google.appengine.tools.pipeline.JobInfo

com.google.appengine.tck.mapreduce.MapReduceTest

com.google.appengine.tck.mapreduce.MapReduceTestBase

com.google.appengine.tools.mapreduce.CustomOutputTest

com.google.appengine.tools.mapreduce.EndToEndTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.