Examples of StreamConfig

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
    if (inputDataSetName != null) {
      // TODO: It's a hack for stream
      if (inputDataSetName.startsWith("stream://")) {
        StreamBatchReadable stream = new StreamBatchReadable(URI.create(inputDataSetName));
        StreamConfig streamConfig = streamAdmin.getConfig(stream.getStreamName());
        Location streamPath = StreamUtils.createGenerationLocation(streamConfig.getLocation(),
                                                                   StreamUtils.getGeneration(streamConfig));

        LOG.info("Using Stream as input from {}", streamPath.toURI());

        TextStreamInputFormat.setTTL(job, streamConfig.getTTL());
        TextStreamInputFormat.setStreamPath(job, streamPath.toURI());
        TextStreamInputFormat.setTimeRange(job, stream.getStartTime(), stream.getEndTime());
        job.setInputFormatClass(TextStreamInputFormat.class);

      } else {
View Full Code Here

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

126
127
128
129
130
131
132
133
134
135
136
            try {
              int currentGeneration = (property == null) ?
                StreamUtils.getGeneration(streamConfig) :
                property.getGeneration();

              StreamConfig newConfig = new StreamConfig(streamConfig.getName(), streamConfig.getPartitionDuration(),
                                                        streamConfig.getIndexInterval(), newTTL,
                                                        streamConfig.getLocation());
              saveConfig(newConfig);
              resultFuture.set(new StreamProperty(currentGeneration, newTTL));
            } catch (IOException e) {
View Full Code Here

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

245
246
247
248
249
250
251
252
253
254
255
256
    private StreamProperty currentProperty;

    private StreamPropertyChangeListener(StreamAdmin streamAdmin, String streamName, StreamPropertyListener listener) {
      this.listener = listener;
      try {
        StreamConfig streamConfig = streamAdmin.getConfig(streamName);
        this.currentProperty = new StreamProperty(StreamUtils.getGeneration(streamConfig), streamConfig.getTTL());
      } catch (Exception e) {
        // It's ok if the stream config is not yet available (meaning no data has ever been writen to the stream yet.
        this.currentProperty = new StreamProperty(0, Long.MAX_VALUE);
      }
    }
View Full Code Here

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

133
134
135
136
137
138
139
140
141
142
143
144
  public void getInfo(HttpRequest request, HttpResponder responder,
                      @PathParam("stream") String stream) throws Exception {
    String accountID = getAuthenticatedAccountId(request);

    if (streamMetaStore.streamExists(accountID, stream)) {
      StreamConfig streamConfig = streamAdmin.getConfig(stream);
      StreamProperties streamProperties = new StreamProperties(streamConfig.getName(), streamConfig.getTTL());
      responder.sendJson(HttpResponseStatus.OK, streamProperties, StreamProperties.class, GSON);
    } else {
      responder.sendStatus(HttpResponseStatus.NOT_FOUND);
    }
  }
View Full Code Here

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

225
226
227
228
229
230
231
232
233
234
235
236
237
238
      responder.sendString(HttpResponseStatus.NOT_FOUND, "Stream does not exists");
      return;
    }

    try {
      StreamConfig config = streamAdmin.getConfig(stream);
      try {
        config = getConfigUpdate(request, config);
        if (config.getTTL() < 0) {
          responder.sendString(HttpResponseStatus.BAD_REQUEST, "TTL value should be positive");
          return;
        }
      } catch (Throwable t) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, "Invalid stream configuration");
View Full Code Here

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

258
259
260
261
262
263
264
265
266
267
268
    // Only pickup changes in TTL
    if (json.has("ttl")) {
      JsonElement ttl = json.get("ttl");
      if (ttl.isJsonPrimitive()) {
        // TTL in the REST API is in seconds. Convert it to ms for the config.
        return new StreamConfig(config.getName(), config.getPartitionDuration(), config.getIndexInterval(),
                                TimeUnit.SECONDS.toMillis(ttl.getAsLong()), config.getLocation());
      }
    }
    return config;
  }
View Full Code Here

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

55
56
57
58
59
60
61
62
63
64
65
66
                                             new DataFabricModules().getDistributedModules(),
                                             new DataSetsModules().getDistributedModule(),
                                             new LocationRuntimeModule().getDistributedModules());

    StreamAdmin streamAdmin = injector.getInstance(StreamAdmin.class);
    StreamConfig streamConfig = streamAdmin.getConfig(streamName);
    Location streamLocation = streamConfig.getLocation();
    List<Location> eventFiles = Lists.newArrayList();

    for (Location partition : streamLocation.list()) {
      if (!partition.isDirectory()) {
        continue;
View Full Code Here

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

260
261
262
263
264
265
266
267
268
269
270
    Supplier<FileWriter<StreamEvent>> create(final String streamName) {
      return new Supplier<FileWriter<StreamEvent>>() {
        @Override
        public FileWriter<StreamEvent> get() {
          try {
            StreamConfig streamConfig = streamAdmin.getConfig(streamName);
            Integer generation = generations.get(streamName);
            if (generation == null) {
              generation = StreamUtils.getGeneration(streamConfig);
            }
View Full Code Here

Examples of co.cask.cdap.data2.transaction.stream.StreamConfig

108
109
110
111
112
113
114
115
116
117
118
119
    if (!verifyGetEventsRequest(accountID, stream, startTime, endTime, limit, responder)) {
      return;
    }

    StreamConfig streamConfig = streamAdmin.getConfig(stream);
    startTime = Math.max(startTime, System.currentTimeMillis() - streamConfig.getTTL());

    // Create the stream event reader
    FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader = createReader(streamConfig, startTime);
    try {
      ReadFilter readFilter = createReadFilter(startTime, endTime);
View Full Code Here

Examples of com.ning.metrics.meteo.binder.StreamConfig

72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
        globalPublisherConfig.setType("com.ning.metrics.meteo.publishers.OpenTSDBListener");
        globalPublisherConfig.setHost("opentsdb.company.com");
        globalPublisherConfig.setPort(4242);
        statementsConfig.setPublishers(Arrays.asList((PublisherConfig) globalPublisherConfig));

        final StreamConfig streamConfig = new StreamConfig();
        streamConfig.setName("TPs of Visit");
        streamConfig.setSql(Arrays.asList("select tp90 from visit output last every 1 second"));
        final HashMap<String, Object> localPublisherConfig = new HashMap<String, Object>();
        localPublisherConfig.put("name", "OpenTSDB");
        localPublisherConfig.put("filters", (Arrays.asList("predict", "tp90")));
        localPublisherConfig.put("timeAttribute", "visit_date");
        streamConfig.setRoutes(Arrays.<HashMap<String, Object>>asList(localPublisherConfig));
        statementsConfig.setStatementConfigs(Arrays.asList(streamConfig));
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.