Examples of Granularity

ch.entwine.weblounge.bridge.oaipmh.Granularity
com.metamx.common.Granularity
com.rackspacecloud.blueflood.rollup.Granularity
1440m [ not enough space to show the relationship, but there would be 6 units of the 240m ranges in 1 1440m range. 240m [ | | ... 60m [ | | | | | | | | ... 20m [ | | | | | | | | | | | | | | | | | | | | | | | | | |... 5m [||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||... full [ granularity to the second, but ranges are partitioned the same as in 5m. ...

Examples of ch.entwine.weblounge.bridge.oaipmh.Granularity

   * <code>baseUrl</code>.
   */
  public static OaiPmhRepositoryClient newHarvester(String baseUrl) {
    OaiPmhRepositoryClient repositoryClient = new OaiPmhRepositoryClient(baseUrl);
    // identify the repository
    Granularity g = repositoryClient.identify().getGranularity();
    repositoryClient.setSupportedGranularity(g);
    return repositoryClient;
  }

View Full Code Here

Examples of com.metamx.common.Granularity

        )
    );


    final Path betaInput = new Path(getInputPath());
    final FileSystem fs = betaInput.getFileSystem(job.getConfiguration());
    final Granularity segmentGranularity = ((UniformGranularitySpec) config.getGranularitySpec()).getSegmentGranularity();


    Map<DateTime, Long> inputModifiedTimes = new TreeMap<DateTime, Long>(
        Comparators.inverse(Comparators.<Comparable>comparable())
    );


    for (FileStatus status : FSSpideringIterator.spiderIterable(fs, betaInput)) {
      final DateTime key = segmentGranularity.toDate(status.getPath().toString());
      final Long currVal = inputModifiedTimes.get(key);
      final long mTime = status.getModificationTime();


      inputModifiedTimes.put(key, currVal == null ? mTime : Math.max(currVal, mTime));
    }


    Set<Interval> bucketsToRun = Sets.newTreeSet(Comparators.intervals());
    for (Map.Entry<DateTime, Long> entry : inputModifiedTimes.entrySet()) {
      DateTime timeBucket = entry.getKey();
      long mTime = entry.getValue();


      String bucketOutput = String.format(
          "%s/%s",
          config.getSchema().getIOConfig().getSegmentOutputPath(),
          segmentGranularity.toPath(timeBucket)
      );
      for (FileStatus fileStatus : FSSpideringIterator.spiderIterable(fs, new Path(bucketOutput))) {
        if (fileStatus.getModificationTime() > mTime) {
          bucketsToRun.add(new Interval(timeBucket, segmentGranularity.increment(timeBucket)));
          break;
        }
      }


      if (bucketsToRun.size() >= maxBuckets) {

View Full Code Here

Examples of com.metamx.common.Granularity

  {
    if (!rejectionPolicy.accept(timestamp)) {
      return null;
    }


    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final VersioningPolicy versioningPolicy = config.getVersioningPolicy();


    final long truncatedTime = segmentGranularity.truncate(new DateTime(timestamp)).getMillis();


    Sink retVal = sinks.get(truncatedTime);


    if (retVal == null) {
      final Interval sinkInterval = new Interval(
          new DateTime(truncatedTime),
          segmentGranularity.increment(new DateTime(truncatedTime))
      );


      retVal = new Sink(sinkInterval, schema, config, versioningPolicy.getVersion(sinkInterval));


      try {

View Full Code Here

Examples of com.metamx.common.Granularity

    }
  }


  protected void startPersistThread()
  {
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final Period windowPeriod = config.getWindowPeriod();


    final DateTime truncatedNow = segmentGranularity.truncate(new DateTime());
    final long windowMillis = windowPeriod.toStandardDuration().getMillis();


    log.info(
        "Expect to run at [%s]",
        new DateTime().plus(
            new Duration(
                System.currentTimeMillis(),
                segmentGranularity.increment(truncatedNow).getMillis() + windowMillis
            )
        )
    );


    ScheduledExecutors
        .scheduleAtFixedRate(
            scheduledExecutor,
            new Duration(
                System.currentTimeMillis(),
                segmentGranularity.increment(truncatedNow).getMillis() + windowMillis
            ),
            new Duration(truncatedNow, segmentGranularity.increment(truncatedNow)),
            new ThreadRenamingCallable<ScheduledExecutors.Signal>(
                String.format(
                    "%s-overseer-%d",
                    schema.getDataSource(),
                    config.getShardSpec().getPartitionNum()

View Full Code Here

Examples of com.metamx.common.Granularity

        );
  }


  private void mergeAndPush()
  {
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final Period windowPeriod = config.getWindowPeriod();


    final long windowMillis = windowPeriod.toStandardDuration().getMillis();
    log.info("Starting merge and push.");
    DateTime minTimestampAsDate = segmentGranularity.truncate(
        new DateTime(
            Math.max(
                windowMillis,
                rejectionPolicy.getCurrMaxTime()
                               .getMillis()

View Full Code Here

Examples of com.metamx.common.Granularity

    );
  }


  private void startFlushThread()
  {
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final DateTime truncatedNow = segmentGranularity.truncate(new DateTime());
    final long windowMillis = config.getWindowPeriod().toStandardDuration().getMillis();


    log.info(
        "Expect to run at [%s]",
        new DateTime().plus(
            new Duration(
                System.currentTimeMillis(),
                schema.getGranularitySpec().getSegmentGranularity().increment(truncatedNow).getMillis() + windowMillis
            )
        )
    );


    ScheduledExecutors
        .scheduleAtFixedRate(
            flushScheduledExec,
            new Duration(
                System.currentTimeMillis(),
                schema.getGranularitySpec().getSegmentGranularity().increment(truncatedNow).getMillis() + windowMillis
            ),
            new Duration(truncatedNow, segmentGranularity.increment(truncatedNow)),
            new ThreadRenamingCallable<ScheduledExecutors.Signal>(
                String.format(
                    "%s-flusher-%d",
                    getSchema().getDataSource(),
                    getConfig().getShardSpec().getPartitionNum()
                )
            )
            {
              @Override
              public ScheduledExecutors.Signal doCall()
              {
                if (stopped) {
                  log.info("Stopping flusher thread");
                  return ScheduledExecutors.Signal.STOP;
                }


                long minTimestamp = segmentGranularity.truncate(
                    getRejectionPolicy().getCurrMaxTime().minus(windowMillis)
                ).getMillis();


                List<Map.Entry<Long, Sink>> sinksToPush = Lists.newArrayList();
                for (Map.Entry<Long, Sink> entry : getSinks().entrySet()) {

View Full Code Here

Examples of com.rackspacecloud.blueflood.rollup.Granularity

        return INSTANCE;
    }


    @Override
    public ByteBuffer toByteBuffer(SlotState state) {
        Granularity gran = state.getGranularity();
        String stringRep = new StringBuilder().append(gran == null ? "null" : gran.name())
                .append(",").append(state.getSlot())
                .append(",").append(state == null ? "null" : state.getState().code())
                .toString();


        return StringSerializer.get().toByteBuffer(stringRep);

View Full Code Here

Examples of com.rackspacecloud.blueflood.rollup.Granularity

    }


    @Override
    public SlotState fromByteBuffer(ByteBuffer byteBuffer) {
        String stringRep = StringSerializer.get().fromByteBuffer(byteBuffer);
        Granularity g = granularityFromStateCol(stringRep);
        Integer slot = slotFromStateCol(stringRep);
        UpdateStamp.State state = stateFromCode(stateCodeFromStateCol(stringRep));


        return new SlotState(g, slot, state);
    }

View Full Code Here

Examples of com.rackspacecloud.blueflood.rollup.Granularity

        Timer.Context ctx = Instrumentation.getWriteTimerContext(CassandraModel.CF_METRICS_STATE);
        try {
            MutationBatch mutationBatch = keyspace.prepareMutationBatch();
            ColumnListMutation<SlotState> mutation = mutationBatch.withRow(CassandraModel.CF_METRICS_STATE, (long)shard);
            for (Map.Entry<Granularity, Map<Integer, UpdateStamp>> granEntry : updates.entrySet()) {
                Granularity g = granEntry.getKey();
                for (Map.Entry<Integer, UpdateStamp> entry : granEntry.getValue().entrySet()) {
                    // granularity,slot,state
                    SlotState slotState = new SlotState(g, entry.getKey(), entry.getValue().getState());
                    mutation.putColumn(slotState, entry.getValue().getTimestamp());
                    /*

View Full Code Here

Examples of com.rackspacecloud.blueflood.rollup.Granularity

import java.nio.ByteBuffer;


public class SlotStateSerializerTest {
    @Test
    public void testGranularityFromStateCol() {
        Granularity myGranularity = SlotStateSerializer.granularityFromStateCol("metrics_full,1,okay");
        Assert.assertNotNull(myGranularity);
        Assert.assertEquals(myGranularity, Granularity.FULL);


        myGranularity = SlotStateSerializer.granularityFromStateCol("FULL");
        Assert.assertNull(myGranularity);

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.