Examples of THostPort

edu.berkeley.sparrow.thrift.THostPort

Examples of edu.berkeley.sparrow.thrift.THostPort


  private TEnqueueTaskReservationsRequest createTaskReservationRequest(
      int numTasks, TaskScheduler scheduler, String userId, int priority) {
    String idStr = Integer.toString(requestId++);
    TUserGroupInfo user = new TUserGroupInfo(userId, "group", priority);
    THostPort schedulerAddress = new THostPort("1.2.3.4", 52);
    return new TEnqueueTaskReservationsRequest(
        "appId", user, idStr, schedulerAddress, numTasks);
  }

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

      // If the scheduler does not enforce a maximum number of tasks, just use a number of
      // threads equal to the number of cores.
      numThreads = Resources.getSystemCPUCount(conf);
    }
    this.scheduler = scheduler;
    nodeMonitorInternalAddress = new THostPort(Network.getIPAddress(conf), nodeMonitorPort);
    ExecutorService service = Executors.newFixedThreadPool(numThreads);
    for (int i = 0; i < numThreads; i++) {
      service.submit(new TaskLaunchRunnable());
    }
  }

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

          LOG.error("Error creating thrift client: " + e.getMessage());
          return;
        }
      }
      BackendService.Client backendClient = backendClients.get(task.appBackendAddress);
      THostPort schedulerHostPort = Network.socketAddressToThrift(task.schedulerAddress);
      TFullTaskId taskId = new TFullTaskId(task.taskSpec.getTaskId(), task.requestId,
          task.appId, schedulerHostPort);
      try {
        backendClient.launchTask(task.taskSpec.bufferForMessage(), taskId, task.user);
      } catch (TException e) {

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

import edu.berkeley.sparrow.thrift.THostPort;


public class Network {
  
  public static THostPort socketAddressToThrift(InetSocketAddress address) {
    return new THostPort(address.getAddress().getHostAddress(), address.getPort());
  }

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

      InetSocketAddress node = nodeList.get(i);
      debugString.append(node.getAddress().getHostAddress() + ":" + node.getPort());
      debugString.append(";");
      // TODO: this needs to be a count!
      outstandingReservations.put(
          new THostPort(node.getAddress().getHostAddress(), node.getPort()),
          numReservations);
      TEnqueueTaskReservationsRequest request = new TEnqueueTaskReservationsRequest(
          schedulingRequest.getApp(), schedulingRequest.getUser(), requestId,
          schedulerAddress, numReservations);
      requests.put(node, request);

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

      TTaskLaunchSpec taskLaunchSpec = new TTaskLaunchSpec(task.getTaskId(),
                                                           task.bufferForMessage());


      int numEnqueuedNodes = 0;
      for (InetSocketAddress addr : preferredNodes) {
        THostPort hostPort = new THostPort(addr.getAddress().getHostAddress(), addr.getPort());
        if (!unlaunchedConstrainedTasks.containsKey(hostPort)) {
          unlaunchedConstrainedTasks.put(
              hostPort, new LinkedList<TTaskLaunchSpec>());
        }

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

  }


  private void populateOutstandingReservations(
      Map<InetSocketAddress, TEnqueueTaskReservationsRequest> requests) {
    for (Entry<InetSocketAddress, TEnqueueTaskReservationsRequest> entry: requests.entrySet()) {
      THostPort hostPort = new THostPort(
          entry.getKey().getAddress().getHostAddress(), entry.getKey().getPort());
      outstandingReservations.put(hostPort, entry.getValue().numTasks);
    }
  }

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

      List<InetSocketAddress> nodes = Lists.newArrayList(requests.keySet());
      assertEquals(nodes.size(), expectedReservations);
      Collections.shuffle(nodes);
      Set<String> taskIdsCopy = Sets.newHashSet(taskIds);
      for (int j = 0; j < expectedReservations; ++j) {
        THostPort hostPort = new THostPort(nodes.get(j).getHostName(), nodes.get(j).getPort());
        List<TTaskLaunchSpec> specs = taskPlacer.assignTask(hostPort);
        assertTrue(specs != null);
        if (j < NUM_TASKS) {
          assertEquals(specs.size(), 1);
          TTaskLaunchSpec spec = specs.get(0);
          assertTrue("Expect to receive a task spec for task " + j + " at " +
                     hostPort.getHost() + ":" + hostPort.getPort(), spec != null);
          assertTrue("Expect list of unlaunched tasks to contain " + spec.getTaskId(),
                     taskIdsCopy.contains(spec.getTaskId()));
          taskIdsCopy.remove(spec.getTaskId());
        } else {
          assertEquals(specs.size(), 0);

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

      Set<String> taskIdsCopy = Sets.newHashSet(taskIds);
      for (int j = 0; j < expectedReservations; ++j) {
        if (j < NUM_TASKS) {
          assertTrue(!taskPlacer.allTasksPlaced());
        }
        THostPort hostPort = new THostPort(nodes.get(j).getHostName(), nodes.get(j).getPort());
        List<TTaskLaunchSpec> specs = taskPlacer.assignTask(hostPort);
        assertTrue(specs != null);
        if (j < NUM_TASKS) {
          assertEquals(1, specs.size());
          TTaskLaunchSpec spec = specs.get(0);
          assertTrue("Expect to receive a task spec for task " + j + " at " +
                     hostPort.getHost() + ":" + hostPort.getPort(), spec != null);
          assertTrue("Expect list of unlaunched tasks to contain " + spec.getTaskId(),
                     taskIdsCopy.contains(spec.getTaskId()));
          taskIdsCopy.remove(spec.getTaskId());
        } else {
          assertTrue(taskPlacer.allTasksPlaced());

View Full Code Here

Examples of edu.berkeley.sparrow.thrift.THostPort

          taskPlacer.getEnqueueTaskReservationsRequests(schedulingRequest, REQUEST_ID, backendNodes,
              SCHEDULER_ADDRESS);


      List<InetSocketAddress> nodes = Lists.newArrayList(requests.keySet());
      for (int taskIndex = 0; taskIndex < NUM_TASKS; ++taskIndex) {
        THostPort hostPort = new THostPort(nodes.get(0).getAddress().getHostAddress(),
                                           nodes.get(0).getPort());
        nodes.remove(0);
        taskPlacer.assignTask(hostPort);
      }


      // Get the node monitors that should be cancelled.  At this point, the remaining entries in
      // nodes should be the set of nodes where requests should be cancelled.
      Set<THostPort> remainingNodeMonitors = taskPlacer.getOutstandingNodeMonitorsForCancellation();
      assertEquals(nodes.size(), remainingNodeMonitors.size());
      for (InetSocketAddress node : nodes) {
        THostPort hostPort = new THostPort(node.getAddress().getHostAddress(), node.getPort());
        assertTrue(remainingNodeMonitors.contains(hostPort));
      }


      // Trying to get the outstanding NMs for cancellation again should return an empty set.
      assertEquals(0, taskPlacer.getOutstandingNodeMonitorsForCancellation().size());

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.