Examples of PipelineService

Examples of com.google.appengine.tools.pipeline.PipelineService

97
98
99
100
101
102
103
104
105
    }

    int entities = parseInt(req.getParameter("entities"));
    int bytesPerEntity = parseInt(req.getParameter("payloadBytesPerEntity"));
    int shardCount = parseInt(req.getParameter("shardCount"));
    PipelineService service = PipelineServiceFactory.newPipelineService();
    redirectToPipelineStatus(resp, service.startNewPipeline(
        new ChainedMapReduceJob(bucket, DATASTORE_TYPE, shardCount, entities, bytesPerEntity)));
  }
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
        .setMapper(new TestMapper()).setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getLongMarshaller())
        .setReducer(ValueProjectionReducer.<String, Long>create())
        .setOutput(new CustomOutput())
        .setNumReducers(17);
    PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
    MapReduceSettings mrSettings = new MapReduceSettings.Builder().build();
    String jobId = pipelineService.startNewPipeline(
        new MapReduceJob<>(mrSpecBuilder.build(), mrSettings));
    assertFalse(jobId.isEmpty());
    executeTasksUntilEmpty("default");
    JobInfo info = pipelineService.getJobInfo(jobId);
    assertEquals(State.COMPLETED_SUCCESSFULLY, info.getJobState());
    @SuppressWarnings("unchecked")
    MapReduceResult<Boolean> result = (MapReduceResult<Boolean>) info.getOutput();
    assertNotNull(result);
    assertTrue(result.getOutputResult());
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

86
87
88
89
90
91
92
93
            + "<br> <a href=index.html>back</a><br>" + "</body></html>");
      }
      return;
    }

    PipelineService service = PipelineServiceFactory.newPipelineService();
    redirectToPipelineStatus(resp, service.startNewPipeline(new BigQueryLoadJob(rowCount)));
  }
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

91
92
93
94
95
96
97
98
99
100
    String module = getStringParam(req, "module", "mapreduce");
    String bucket = getBucketParam(req);
    long start = getLongParam(req, "start", 0);
    long limit = getLongParam(req, "limit", 100 * 1000 * 1000);
    int shards = Math.max(1, Math.min(100, Ints.saturatedCast(getLongParam(req, "shards", 30))));
    PipelineService service = PipelineServiceFactory.newPipelineService();
    String pipelineId = service.startNewPipeline(new MyPipelineJob(bucket, start, limit, shards),
        new JobSetting.OnQueue(queue), new JobSetting.OnModule(module));
    resp.sendRedirect("/_ah/pipeline/status.html?root=" + pipelineId);
  }
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

57
58
59
60
61
62
63
64
65
  public static <I, O, R> String start(MapSpecification<I, O, R> specification,
      MapSettings settings) {
    if (settings.getWorkerQueueName() == null) {
      settings = new MapSettings.Builder(settings).setWorkerQueueName("default").build();
    }
    PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
    return pipelineService.startNewPipeline(
        new MapJob<>(specification, settings), settings.toJobSettings());
  }
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

38
39
40
41
42
43
44
45
    // TODO(user): should not be needed once b/9940384 is fixed
    return futureCall(new DeletePipelineJob(getPipelineKey().getName()), settings);
  }

  public static void cleanup(List<GcsFilename> toDelete, JobSetting... settings) {
    PipelineService service = PipelineServiceFactory.newPipelineService();
    service.startNewPipeline(new CleanupPipelineJob(), toDelete, settings);
  }
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
    DeferredTask deleteRecordsTask = new DeferredTask() {
      private static final long serialVersionUID = -7510918963650055768L;

      @Override
      public void run() {
        PipelineService service = PipelineServiceFactory.newPipelineService();
        try {
          service.deletePipelineRecords(key);
          log.info("Deleted pipeline: " + key);
        } catch (IllegalStateException e) {
          log.info("Failed to delete pipeline: " + key);
          HttpServletRequest request = DeferredTaskContext.getCurrentRequest();
          if (request != null) {
            int attempts = request.getIntHeader("X-AppEngine-TaskExecutionCount");
            if (attempts <= 5) {
              log.info("Request to retry deferred task #" + attempts);
              DeferredTaskContext.markForRetry();
              return;
            }
          }
          try {
            service.deletePipelineRecords(key, true, false);
            log.info("Force deleted pipeline: " + key);
          } catch (Exception ex) {
            log.log(Level.WARNING, "Failed to force delete pipeline: " + key, ex);
          }
        } catch (NoSuchObjectException e) {
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

87
88
89
90
91
92
93
94
95
  public static <I, K, V, O, R> String start(
      MapReduceSpecification<I, K, V, O, R> specification, MapReduceSettings settings) {
    if (settings.getWorkerQueueName() == null) {
      settings = new MapReduceSettings.Builder(settings).setWorkerQueueName("default").build();
    }
    PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
    return pipelineService.startNewPipeline(
        new MapReduceJob<>(specification, settings), settings.toJobSettings());
  }
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

276
277
278
279
280
281
282
283
284
285
        callCompleted(jobState);
      } else {
        log.info("Calling failed for " + jobId + ", status=" + jobState.getStatus());
        jobState.getController().failed(jobState.getStatus());
      }
      PipelineService pipeline = PipelineServiceFactory.newPipelineService();
      pipeline.startNewPipeline(
          new FinalizeShardedJob(jobId, jobState.getTotalTaskCount(), jobState.getStatus()));
    }
  }
View Full Code Here

Examples of com.google.appengine.tools.pipeline.PipelineService

662
663
664
665
666
667
668
669
670
671
672
673
    if (jobState.getStatus().isActive()) {
      return false;
    }
    int taskCount = jobState.getTotalTaskCount();
    if (taskCount > 0) {
      PipelineService pipeline = PipelineServiceFactory.newPipelineService();
      pipeline.startNewPipeline(new DeleteShardedJob(jobId, taskCount));
    }
    final Key jobKey = ShardedJobStateImpl.ShardedJobSerializer.makeKey(jobId);
    RetryHelper.runWithRetries(callable(new Runnable() {
      @Override
      public void run() {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.