Package com.linkedin.camus.etl.kafka.mapred

Examples of com.linkedin.camus.etl.kafka.mapred.EtlSplit


    List<InputSplit> kafkaETLSplits = new ArrayList<InputSplit>();

    for (int i = 0; i < numTasks; i++) {
      if (requests.size() > 0) {
        kafkaETLSplits.add(new EtlSplit());
      }
    }

    for (CamusRequest r : requests) {
      getSmallestMultiSplit(kafkaETLSplits).addRequest(r);
View Full Code Here


    return kafkaETLSplits;
  }
 
  protected EtlSplit getSmallestMultiSplit(List<InputSplit> kafkaETLSplits)
      throws IOException {
    EtlSplit smallest = (EtlSplit) kafkaETLSplits.get(0);

    for (int i = 1; i < kafkaETLSplits.size(); i++) {
      EtlSplit challenger = (EtlSplit) kafkaETLSplits.get(i);
      if ((smallest.getLength() == challenger.getLength() && smallest
          .getNumRequests() > challenger.getNumRequests())
          || smallest.getLength() > challenger.getLength()) {
        smallest = challenger;
      }
    }

    return smallest;
View Full Code Here

        .getInt("mapred.map.tasks", 30);
    List<InputSplit> kafkaETLSplits = new ArrayList<InputSplit>();
   
    for (int i = 0; i < numTasks; i++) {
      if (requests.size() > 0) {
        kafkaETLSplits.add(new EtlSplit());
      }
    }
   
    List<CamusRequest> groupedRequests = groupSmallRequest(requests, context);
   
    reverseSortRequests(groupedRequests);

    for (CamusRequest r : groupedRequests) {
      EtlSplit split = getSmallestMultiSplit(kafkaETLSplits);
      for (CamusRequest r1 : (GroupedRequest)r) {
        split.addRequest(r1);
      }
    }

    return kafkaETLSplits;
  }
View Full Code Here

TOP

Related Classes of com.linkedin.camus.etl.kafka.mapred.EtlSplit

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.