Package org.apache.hadoop.yarn.api.records.timeline

Examples of org.apache.hadoop.yarn.api.records.timeline.TimelineEntities


    Assert.assertEquals(0, event2.getEventInfo().size());
  }

  @Test
  public void testPostEntitiesWithPrimaryFilter() throws Exception {
    TimelineEntities entities = new TimelineEntities();
    TimelineEntity entity = new TimelineEntity();
    Map<String, Set<Object>> filters = new HashMap<String, Set<Object>>();
    filters.put(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), new HashSet<Object>());
    entity.setPrimaryFilters(filters);
    entity.setEntityId("test id 6");
    entity.setEntityType("test type 6");
    entity.setStartTime(System.currentTimeMillis());
    entities.addEntity(entity);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("timeline")
        .accept(MediaType.APPLICATION_JSON)
        .type(MediaType.APPLICATION_JSON)
        .post(ClientResponse.class, entities);
View Full Code Here


      errors.get(0).getErrorCode());
  }
 
  @Test
  public void testPostEntities() throws Exception {
    TimelineEntities entities = new TimelineEntities();
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId("test id 1");
    entity.setEntityType("test type 1");
    entity.setStartTime(System.currentTimeMillis());
    entities.addEntity(entity);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("timeline")
        .accept(MediaType.APPLICATION_JSON)
        .type(MediaType.APPLICATION_JSON)
        .post(ClientResponse.class, entities);
View Full Code Here

  public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
    AdminACLsManager oldAdminACLsManager =
        timelineACLsManager.setAdminACLsManager(adminACLsManager);
    remoteUser = "tester";
    try {
      TimelineEntities entities = new TimelineEntities();
      TimelineEntity entity = new TimelineEntity();
      entity.setEntityId("test id 2");
      entity.setEntityType("test type 2");
      entity.setStartTime(System.currentTimeMillis());
      entities.addEntity(entity);
      WebResource r = resource();
      ClientResponse response = r.path("ws").path("v1").path("timeline")
          .accept(MediaType.APPLICATION_JSON)
          .type(MediaType.APPLICATION_JSON)
          .post(ClientResponse.class, entities);
View Full Code Here

  public void testGetEntityWithYarnACLsEnabled() throws Exception {
    AdminACLsManager oldAdminACLsManager =
        timelineACLsManager.setAdminACLsManager(adminACLsManager);
    remoteUser = "tester";
    try {
      TimelineEntities entities = new TimelineEntities();
      TimelineEntity entity = new TimelineEntity();
      entity.setEntityId("test id 3");
      entity.setEntityType("test type 3");
      entity.setStartTime(System.currentTimeMillis());
      entities.addEntity(entity);
      WebResource r = resource();
      ClientResponse response = r.path("ws").path("v1").path("timeline")
          .accept(MediaType.APPLICATION_JSON)
          .type(MediaType.APPLICATION_JSON)
          .post(ClientResponse.class, entities);
View Full Code Here

  public void testGetEntitiesWithYarnACLsEnabled() {
    AdminACLsManager oldAdminACLsManager =
        timelineACLsManager.setAdminACLsManager(adminACLsManager);
    remoteUser = "tester";
    try {
      TimelineEntities entities = new TimelineEntities();
      TimelineEntity entity = new TimelineEntity();
      entity.setEntityId("test id 4");
      entity.setEntityType("test type 4");
      entity.setStartTime(System.currentTimeMillis());
      entities.addEntity(entity);
      WebResource r = resource();
      ClientResponse response = r.path("ws").path("v1").path("timeline")
          .accept(MediaType.APPLICATION_JSON)
          .type(MediaType.APPLICATION_JSON)
          .post(ClientResponse.class, entities);

      remoteUser = "other";
      entities = new TimelineEntities();
      entity = new TimelineEntity();
      entity.setEntityId("test id 5");
      entity.setEntityType("test type 4");
      entity.setStartTime(System.currentTimeMillis());
      entities.addEntity(entity);
      r = resource();
      response = r.path("ws").path("v1").path("timeline")
          .accept(MediaType.APPLICATION_JSON)
          .type(MediaType.APPLICATION_JSON)
          .post(ClientResponse.class, entities);

      response = r.path("ws").path("v1").path("timeline")
          .path("test type 4")
          .accept(MediaType.APPLICATION_JSON)
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      entities = response.getEntity(TimelineEntities.class);
      assertEquals(1, entities.getEntities().size());
      assertEquals("test type 4", entities.getEntities().get(0).getEntityType());
      assertEquals("test id 5", entities.getEntities().get(0).getEntityId());
    } finally {
      timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
      remoteUser = null;
    }
  }
View Full Code Here

  public void testGetEventsWithYarnACLsEnabled() {
    AdminACLsManager oldAdminACLsManager =
        timelineACLsManager.setAdminACLsManager(adminACLsManager);
    remoteUser = "tester";
    try {
      TimelineEntities entities = new TimelineEntities();
      TimelineEntity entity = new TimelineEntity();
      entity.setEntityId("test id 5");
      entity.setEntityType("test type 5");
      entity.setStartTime(System.currentTimeMillis());
      TimelineEvent event = new TimelineEvent();
      event.setEventType("event type 1");
      event.setTimestamp(System.currentTimeMillis());
      entity.addEvent(event);
      entities.addEntity(entity);
      WebResource r = resource();
      ClientResponse response = r.path("ws").path("v1").path("timeline")
          .accept(MediaType.APPLICATION_JSON)
          .type(MediaType.APPLICATION_JSON)
          .post(ClientResponse.class, entities);

      remoteUser = "other";
      entities = new TimelineEntities();
      entity = new TimelineEntity();
      entity.setEntityId("test id 6");
      entity.setEntityType("test type 5");
      entity.setStartTime(System.currentTimeMillis());
      event = new TimelineEvent();
      event.setEventType("event type 2");
      event.setTimestamp(System.currentTimeMillis());
      entity.addEvent(event);
      entities.addEntity(entity);
      r = resource();
      response = r.path("ws").path("v1").path("timeline")
          .accept(MediaType.APPLICATION_JSON)
          .type(MediaType.APPLICATION_JSON)
          .post(ClientResponse.class, entities);
View Full Code Here

    Iterator<TimelineEntity> entityIterator = null;
    if (fromId != null) {
      TimelineEntity firstEntity = entities.get(new EntityIdentifier(fromId,
          entityType));
      if (firstEntity == null) {
        return new TimelineEntities();
      } else {
        entityIterator = new TreeSet<TimelineEntity>(entities.values())
            .tailSet(firstEntity, true).iterator();
      }
    }
    if (entityIterator == null) {
      entityIterator = new PriorityQueue<TimelineEntity>(entities.values())
          .iterator();
    }

    List<TimelineEntity> entitiesSelected = new ArrayList<TimelineEntity>();
    while (entityIterator.hasNext()) {
      TimelineEntity entity = entityIterator.next();
      if (entitiesSelected.size() >= limit) {
        break;
      }
      if (!entity.getEntityType().equals(entityType)) {
        continue;
      }
      if (entity.getStartTime() <= windowStart) {
        continue;
      }
      if (entity.getStartTime() > windowEnd) {
        continue;
      }
      if (fromTs != null && entityInsertTimes.get(new EntityIdentifier(
          entity.getEntityId(), entity.getEntityType())) > fromTs) {
        continue;
      }
      if (primaryFilter != null &&
          !matchPrimaryFilter(entity.getPrimaryFilters(), primaryFilter)) {
        continue;
      }
      if (secondaryFilters != null) { // AND logic
        boolean flag = true;
        for (NameValuePair secondaryFilter : secondaryFilters) {
          if (secondaryFilter != null && !matchPrimaryFilter(
              entity.getPrimaryFilters(), secondaryFilter) &&
              !matchFilter(entity.getOtherInfo(), secondaryFilter)) {
            flag = false;
            break;
          }
        }
        if (!flag) {
          continue;
        }
      }
      entitiesSelected.add(entity);
    }
    List<TimelineEntity> entitiesToReturn = new ArrayList<TimelineEntity>();
    for (TimelineEntity entitySelected : entitiesSelected) {
      entitiesToReturn.add(maskFields(entitySelected, fields));
    }
    Collections.sort(entitiesToReturn);
    TimelineEntities entitiesWrapper = new TimelineEntities();
    entitiesWrapper.setEntities(entitiesToReturn);
    return entitiesWrapper;
  }
View Full Code Here

      byte[] first = null;
      if (fromId != null) {
        Long fromIdStartTime = getStartTimeLong(fromId, entityType);
        if (fromIdStartTime == null) {
          // no start time for provided id, so return empty entities
          return new TimelineEntities();
        }
        if (fromIdStartTime <= endtime) {
          // if provided id's start time falls before the end of the window,
          // use it to construct the seek key
          first = kb.add(writeReverseOrderedLong(fromIdStartTime))
              .add(fromId).getBytesForLookup();
        }
      }
      // if seek key wasn't constructed using fromId, construct it using end ts
      if (first == null) {
        first = kb.add(writeReverseOrderedLong(endtime)).getBytesForLookup();
      }
      byte[] last = null;
      if (starttime != null) {
        // if start time is not null, set a last key that will not be
        // iterated past
        last = KeyBuilder.newInstance().add(base).add(entityType)
            .add(writeReverseOrderedLong(starttime)).getBytesForLookup();
      }
      if (limit == null) {
        // if limit is not specified, use the default
        limit = DEFAULT_LIMIT;
      }

      TimelineEntities entities = new TimelineEntities();
      iterator = db.iterator();
      iterator.seek(first);
      // iterate until one of the following conditions is met: limit is
      // reached, there are no more keys, the key prefix no longer matches,
      // or a start time has been specified and reached/exceeded
      while (entities.getEntities().size() < limit && iterator.hasNext()) {
        byte[] key = iterator.peekNext().getKey();
        if (!prefixMatches(prefix, prefix.length, key) || (last != null &&
            WritableComparator.compareBytes(key, 0, key.length, last, 0,
                last.length) > 0)) {
          break;
        }
        // read the start time and entity id from the current key
        KeyParser kp = new KeyParser(key, prefix.length);
        Long startTime = kp.getNextLong();
        String entityId = kp.getNextString();

        if (fromTs != null) {
          long insertTime = readReverseOrderedLong(iterator.peekNext()
              .getValue(), 0);
          if (insertTime > fromTs) {
            byte[] firstKey = key;
            while (iterator.hasNext() && prefixMatches(firstKey,
                kp.getOffset(), key)) {
              iterator.next();
              key = iterator.peekNext().getKey();
            }
            continue;
          }
        }

        // parse the entity that owns this key, iterating over all keys for
        // the entity
        TimelineEntity entity = getEntity(entityId, entityType, startTime,
            fields, iterator, key, kp.getOffset());
        // determine if the retrieved entity matches the provided secondary
        // filters, and if so add it to the list of entities to return
        boolean filterPassed = true;
        if (secondaryFilters != null) {
          for (NameValuePair filter : secondaryFilters) {
            Object v = entity.getOtherInfo().get(filter.getName());
            if (v == null) {
              Set<Object> vs = entity.getPrimaryFilters()
                  .get(filter.getName());
              if (vs != null && !vs.contains(filter.getValue())) {
                filterPassed = false;
                break;
              }
            } else if (!v.equals(filter.getValue())) {
              filterPassed = false;
              break;
            }
          }
        }
        if (filterPassed) {
          entities.addEntity(entity);
        }
      }
      return entities;
    } finally {
      IOUtils.cleanup(LOG, iterator);
View Full Code Here

      @QueryParam("fromId") String fromId,
      @QueryParam("fromTs") String fromTs,
      @QueryParam("limit") String limit,
      @QueryParam("fields") String fields) {
    init(res);
    TimelineEntities entities = null;
    try {
      EnumSet<Field> fieldEnums = parseFieldsStr(fields, ",");
      boolean modified = extendFields(fieldEnums);
      UserGroupInformation callerUGI = getUser(req);
      entities = store.getEntities(
          parseStr(entityType),
          parseLongStr(limit),
          parseLongStr(windowStart),
          parseLongStr(windowEnd),
          parseStr(fromId),
          parseLongStr(fromTs),
          parsePairStr(primaryFilter, ":"),
          parsePairsStr(secondaryFilter, ",", ":"),
          fieldEnums);
      if (entities != null) {
        Iterator<TimelineEntity> entitiesItr =
            entities.getEntities().iterator();
        while (entitiesItr.hasNext()) {
          TimelineEntity entity = entitiesItr.next();
          try {
            // check ACLs
            if (!timelineACLsManager.checkAccess(callerUGI, entity)) {
              entitiesItr.remove();
            } else {
              // clean up system data
              if (modified) {
                entity.setPrimaryFilters(null);
              } else {
                cleanupOwnerInfo(entity);
              }
            }
          } catch (YarnException e) {
            LOG.error("Error when verifying access for user " + callerUGI
                + " on the events of the timeline entity "
                + new EntityIdentifier(entity.getEntityId(),
                    entity.getEntityType()), e);
            entitiesItr.remove();
          }
        }
      }
    } catch (NumberFormatException e) {
      throw new BadRequestException(
          "windowStart, windowEnd or limit is not a numeric value.");
    } catch (IllegalArgumentException e) {
      throw new BadRequestException("requested invalid field.");
    } catch (IOException e) {
      LOG.error("Error getting entities", e);
      throw new WebApplicationException(e,
          Response.Status.INTERNAL_SERVER_ERROR);
    }
    if (entities == null) {
      return new TimelineEntities();
    }
    return entities;
  }
View Full Code Here

      return new TimelinePutResponse();
    }
    UserGroupInformation callerUGI = getUser(req);
    try {
      List<EntityIdentifier> entityIDs = new ArrayList<EntityIdentifier>();
      TimelineEntities entitiesToPut = new TimelineEntities();
      List<TimelinePutResponse.TimelinePutError> errors =
          new ArrayList<TimelinePutResponse.TimelinePutError>();
      for (TimelineEntity entity : entities.getEntities()) {
        EntityIdentifier entityID =
            new EntityIdentifier(entity.getEntityId(), entity.getEntityType());

        // check if there is existing entity
        TimelineEntity existingEntity = null;
        try {
          existingEntity =
              store.getEntity(entityID.getId(), entityID.getType(),
                  EnumSet.of(Field.PRIMARY_FILTERS));
          if (existingEntity != null
              && !timelineACLsManager.checkAccess(callerUGI, existingEntity)) {
            throw new YarnException("The timeline entity " + entityID
                + " was not put by " + callerUGI + " before");
          }
        } catch (Exception e) {
          // Skip the entity which already exists and was put by others
          LOG.warn("Skip the timeline entity: " + entityID + ", because "
              + e.getMessage());
          TimelinePutResponse.TimelinePutError error =
              new TimelinePutResponse.TimelinePutError();
          error.setEntityId(entityID.getId());
          error.setEntityType(entityID.getType());
          error.setErrorCode(
              TimelinePutResponse.TimelinePutError.ACCESS_DENIED);
          errors.add(error);
          continue;
        }

        // inject owner information for the access check if this is the first
        // time to post the entity, in case it's the admin who is updating
        // the timeline data.
        try {
          if (existingEntity == null) {
            injectOwnerInfo(entity,
                callerUGI == null ? "" : callerUGI.getShortUserName());
          }
        } catch (YarnException e) {
          // Skip the entity which messes up the primary filter and record the
          // error
          LOG.warn("Skip the timeline entity: " + entityID + ", because "
              + e.getMessage());
          TimelinePutResponse.TimelinePutError error =
              new TimelinePutResponse.TimelinePutError();
          error.setEntityId(entityID.getId());
          error.setEntityType(entityID.getType());
          error.setErrorCode(
              TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT);
          errors.add(error);
          continue;
        }

        entityIDs.add(entityID);
        entitiesToPut.addEntity(entity);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Storing the entity " + entityID + ", JSON-style content: "
              + TimelineUtils.dumpTimelineRecordtoJSON(entity));
        }
      }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.yarn.api.records.timeline.TimelineEntities

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.