Examples of MergeStats


Examples of org.apache.accumulo.master.state.MergeStats

      try {
        Map<Text,MergeStats> mergeStatsCache = new HashMap<Text,MergeStats>();
        Map<Text,MergeStats> currentMerges = new HashMap<Text,MergeStats>();
        for (MergeInfo merge : master.merges()) {
          if (merge.getExtent() != null) {
            currentMerges.put(merge.getExtent().getTableId(), new MergeStats(merge));
          }
        }
       
        // Get the current status for the current list of tservers
        SortedMap<TServerInstance,TabletServerStatus> currentTServers = new TreeMap<TServerInstance,TabletServerStatus>();
        for (TServerInstance entry : this.master.tserverSet.getCurrentServers()) {
          currentTServers.put(entry, this.master.tserverStatus.get(entry));
        }
       
        if (currentTServers.size() == 0) {
          eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
          continue;
        }
       
        // Don't move tablets to servers that are shutting down
        SortedMap<TServerInstance,TabletServerStatus> destinations = new TreeMap<TServerInstance,TabletServerStatus>(currentTServers);
        destinations.keySet().removeAll(this.master.serversToShutdown);
       
        List<Assignment> assignments = new ArrayList<Assignment>();
        List<Assignment> assigned = new ArrayList<Assignment>();
        List<TabletLocationState> assignedToDeadServers = new ArrayList<TabletLocationState>();
        Map<KeyExtent,TServerInstance> unassigned = new HashMap<KeyExtent,TServerInstance>();
       
        int[] counts = new int[TabletState.values().length];
        stats.begin();
        // Walk through the tablets in our store, and work tablets
        // towards their goal
        iter = store.iterator();
        while (iter.hasNext()) {
          TabletLocationState tls = iter.next();
          if (tls == null) {
            continue;
          }
          // ignore entries for tables that do not exist in zookeeper
          if (TableManager.getInstance().getTableState(tls.extent.getTableId().toString()) == null)
            continue;
         
          if (Master.log.isTraceEnabled())
            Master.log.trace(tls + " walogs " + tls.walogs.size());
                   
          // Don't overwhelm the tablet servers with work
          if (unassigned.size() + unloaded > Master.MAX_TSERVER_WORK_CHUNK * currentTServers.size()) {
            flushChanges(destinations, assignments, assigned, assignedToDeadServers, unassigned);
            assignments.clear();
            assigned.clear();
            assignedToDeadServers.clear();
            unassigned.clear();
            unloaded = 0;
            eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
          }
          Text tableId = tls.extent.getTableId();
          MergeStats mergeStats = mergeStatsCache.get(tableId);
          if (mergeStats == null) {
            mergeStats = currentMerges.get(tableId);
            if (mergeStats == null) {
              mergeStats = new MergeStats(new MergeInfo());
            }
            mergeStatsCache.put(tableId, mergeStats);
          }
          TabletGoalState goal = this.master.getGoalState(tls, mergeStats.getMergeInfo());
          TServerInstance server = tls.getServer();
          TabletState state = tls.getState(currentTServers.keySet());
          if (Master.log.isTraceEnabled())
            Master.log.trace("Goal state " + goal + " current " + state);
          stats.update(tableId, state);
          mergeStats.update(tls.extent, state, tls.chopped, !tls.walogs.isEmpty());
          sendChopRequest(mergeStats.getMergeInfo(), state, tls);
          sendSplitRequest(mergeStats.getMergeInfo(), state, tls);
         
          // Always follow through with assignments
          if (state == TabletState.ASSIGNED) {
            goal = TabletGoalState.HOSTED;
          }
View Full Code Here

Examples of org.apache.accumulo.master.state.MergeStats

    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
    TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(new Text("o")));
    update(connector, m);

    // do the state check
    MergeStats stats = scan(state, metaDataStateStore);
    MergeState newState = stats.nextMergeState(connector, state);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, newState);

    // unassign the tablets
    BatchDeleter deleter = connector.createBatchDeleter(MetadataTable.NAME, Authorizations.EMPTY, 1000, new BatchWriterConfig());
    deleter.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
    deleter.setRanges(Collections.singletonList(new Range()));
    deleter.delete();

    // now we should be ready to merge but, we have inconsistent metadata
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));

    // finish the split
    KeyExtent tablet = new KeyExtent(tableId, new Text("p"), new Text("o"));
    m = tablet.getPrevRowUpdateMutation();
    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
    update(connector, m);
    metaDataStateStore.setLocations(Collections.singletonList(new Assignment(tablet, state.someTServer)));

    // onos... there's a new tablet online
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState(connector, state));

    // chop it
    m = tablet.getPrevRowUpdateMutation();
    ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("junk".getBytes()));
    update(connector, m);

    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));

    // take it offline
    m = tablet.getPrevRowUpdateMutation();
    Collection<Collection<String>> walogs = Collections.emptyList();
    metaDataStateStore.unassign(Collections.singletonList(new TabletLocationState(tablet, null, state.someTServer, null, walogs, false)));

    // now we can split
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.MERGING, stats.nextMergeState(connector, state));

  }
View Full Code Here

Examples of org.apache.accumulo.master.state.MergeStats

    Assert.assertEquals(MergeState.MERGING, stats.nextMergeState(connector, state));

  }

  private MergeStats scan(MockCurrentState state, MetaDataStateStore metaDataStateStore) {
    MergeStats stats = new MergeStats(state.mergeInfo);
    stats.getMergeInfo().setState(MergeState.WAITING_FOR_OFFLINE);
    for (TabletLocationState tss : metaDataStateStore) {
      stats.update(tss.extent, tss.getState(state.onlineTabletServers()), tss.chopped, false);
    }
    return stats;
  }
View Full Code Here

Examples of org.apache.accumulo.master.state.MergeStats

      try {
        Map<Text,MergeStats> mergeStatsCache = new HashMap<Text,MergeStats>();
        Map<Text,MergeStats> currentMerges = new HashMap<Text,MergeStats>();
        for (MergeInfo merge : master.merges()) {
          if (merge.getExtent() != null) {
            currentMerges.put(merge.getExtent().getTableId(), new MergeStats(merge));
          }
        }
       
        // Get the current status for the current list of tservers
        SortedMap<TServerInstance,TabletServerStatus> currentTServers = new TreeMap<TServerInstance,TabletServerStatus>();
        for (TServerInstance entry : this.master.tserverSet.getCurrentServers()) {
          currentTServers.put(entry, this.master.tserverStatus.get(entry));
        }
       
        if (currentTServers.size() == 0) {
          eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
          continue;
        }
       
        // Don't move tablets to servers that are shutting down
        SortedMap<TServerInstance,TabletServerStatus> destinations = new TreeMap<TServerInstance,TabletServerStatus>(currentTServers);
        destinations.keySet().removeAll(this.master.serversToShutdown);
       
        List<Assignment> assignments = new ArrayList<Assignment>();
        List<Assignment> assigned = new ArrayList<Assignment>();
        List<TabletLocationState> assignedToDeadServers = new ArrayList<TabletLocationState>();
        Map<KeyExtent,TServerInstance> unassigned = new HashMap<KeyExtent,TServerInstance>();
       
        int[] counts = new int[TabletState.values().length];
        stats.begin();
        // Walk through the tablets in our store, and work tablets
        // towards their goal
        iter = store.iterator();
        while (iter.hasNext()) {
          TabletLocationState tls = iter.next();
          if (tls == null) {
            continue;
          }
          // ignore entries for tables that do not exist in zookeeper
          if (TableManager.getInstance().getTableState(tls.extent.getTableId().toString()) == null)
            continue;
         
          if (Master.log.isTraceEnabled())
            Master.log.trace(tls + " walogs " + tls.walogs.size());
                   
          // Don't overwhelm the tablet servers with work
          if (unassigned.size() + unloaded > Master.MAX_TSERVER_WORK_CHUNK * currentTServers.size()) {
            flushChanges(destinations, assignments, assigned, assignedToDeadServers, unassigned);
            assignments.clear();
            assigned.clear();
            assignedToDeadServers.clear();
            unassigned.clear();
            unloaded = 0;
            eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
          }
          Text tableId = tls.extent.getTableId();
          MergeStats mergeStats = mergeStatsCache.get(tableId);
          if (mergeStats == null) {
            mergeStats = currentMerges.get(tableId);
            if (mergeStats == null) {
              mergeStats = new MergeStats(new MergeInfo());
            }
            mergeStatsCache.put(tableId, mergeStats);
          }
          TabletGoalState goal = this.master.getGoalState(tls, mergeStats.getMergeInfo());
          TServerInstance server = tls.getServer();
          TabletState state = tls.getState(currentTServers.keySet());
          if (Master.log.isTraceEnabled())
            Master.log.trace("Goal state " + goal + " current " + state);
          stats.update(tableId, state);
          mergeStats.update(tls.extent, state, tls.chopped, !tls.walogs.isEmpty());
          sendChopRequest(mergeStats.getMergeInfo(), state, tls);
          sendSplitRequest(mergeStats.getMergeInfo(), state, tls);
         
          // Always follow through with assignments
          if (state == TabletState.ASSIGNED) {
            goal = TabletGoalState.HOSTED;
          }
View Full Code Here

Examples of org.apache.accumulo.master.state.MergeStats

    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
    TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(new Text("o")));
    update(connector, m);

    // do the state check
    MergeStats stats = scan(state, metaDataStateStore);
    MergeState newState = stats.nextMergeState(connector, state);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, newState);

    // unassign the tablets
    BatchDeleter deleter = connector.createBatchDeleter(MetadataTable.NAME, Authorizations.EMPTY, 1000, new BatchWriterConfig());
    deleter.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
    deleter.setRanges(Collections.singletonList(new Range()));
    deleter.delete();

    // now we should be ready to merge but, we have inconsistent metadata
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));

    // finish the split
    KeyExtent tablet = new KeyExtent(tableId, new Text("p"), new Text("o"));
    m = tablet.getPrevRowUpdateMutation();
    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
    update(connector, m);
    metaDataStateStore.setLocations(Collections.singletonList(new Assignment(tablet, state.someTServer)));

    // onos... there's a new tablet online
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState(connector, state));

    // chop it
    m = tablet.getPrevRowUpdateMutation();
    ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("junk".getBytes()));
    update(connector, m);

    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));

    // take it offline
    m = tablet.getPrevRowUpdateMutation();
    Collection<Collection<String>> walogs = Collections.emptyList();
    metaDataStateStore.unassign(Collections.singletonList(new TabletLocationState(tablet, null, state.someTServer, null, walogs, false)));

    // now we can split
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.MERGING, stats.nextMergeState(connector, state));

  }
View Full Code Here

Examples of org.apache.accumulo.master.state.MergeStats

   * @param metaDataStateStore
   * @param locations
   * @return
   */
  private MergeStats scan(MockCurrentState state, MetaDataStateStore metaDataStateStore) {
    MergeStats stats = new MergeStats(state.mergeInfo);
    stats.getMergeInfo().setState(MergeState.WAITING_FOR_OFFLINE);
    for (TabletLocationState tss : metaDataStateStore) {
      stats.update(tss.extent, tss.getState(state.onlineTabletServers()), tss.chopped, false);
    }
    return stats;
  }
View Full Code Here

Examples of org.apache.accumulo.server.master.state.MergeStats

    Constants.METADATA_SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
    Constants.METADATA_OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(new Text("o")));
    update(connector, m);
   
    // do the state check
    MergeStats stats = scan(state, metaDataStateStore);
    MergeState newState = stats.nextMergeState(connector, state);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, newState);
   
    // unassign the tablets
    BatchDeleter deleter = connector.createBatchDeleter("!METADATA", Constants.NO_AUTHS, 1000, new BatchWriterConfig());
    deleter.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
    deleter.setRanges(Collections.singletonList(new Range()));
    deleter.delete();
   
    // now we should be ready to merge but, we have an inconsistent !METADATA table
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));
   
    // finish the split
    KeyExtent tablet = new KeyExtent(tableId, new Text("p"), new Text("o"));
    m = tablet.getPrevRowUpdateMutation();
    Constants.METADATA_SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
    update(connector, m);
    metaDataStateStore.setLocations(Collections.singletonList(new Assignment(tablet, state.someTServer)));
   
    // onos... there's a new tablet online
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState(connector, state));
   
    // chop it
    m = tablet.getPrevRowUpdateMutation();
    Constants.METADATA_CHOPPED_COLUMN.put(m, new Value("junk".getBytes()));
    update(connector, m);
   
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));
   
    // take it offline
    m = tablet.getPrevRowUpdateMutation();
    Collection<Collection<String>> walogs = Collections.emptyList();
    metaDataStateStore.unassign(Collections.singletonList(new TabletLocationState(tablet, null, state.someTServer, null, walogs, false)));
   
    // now we can split
    stats = scan(state, metaDataStateStore);
    Assert.assertEquals(MergeState.MERGING, stats.nextMergeState(connector, state));
   
  }
View Full Code Here

Examples of org.apache.accumulo.server.master.state.MergeStats

    Assert.assertEquals(MergeState.MERGING, stats.nextMergeState(connector, state));
   
  }
 
  private MergeStats scan(MockCurrentState state, MetaDataStateStore metaDataStateStore) {
    MergeStats stats = new MergeStats(state.mergeInfo);
    stats.getMergeInfo().setState(MergeState.WAITING_FOR_OFFLINE);
    for (TabletLocationState tss : metaDataStateStore) {
      stats.update(tss.extent, tss.getState(state.onlineTabletServers()), tss.chopped, false);
    }
    return stats;
  }
View Full Code Here

Examples of org.apache.accumulo.server.master.state.MergeStats

        try {
          Map<Text,MergeStats> mergeStatsCache = new HashMap<Text,MergeStats>();
          Map<Text,MergeStats> currentMerges = new HashMap<Text,MergeStats>();
          for (MergeInfo merge : merges()) {
            if (merge.getRange() != null) {
              currentMerges.put(merge.getRange().getTableId(), new MergeStats(merge));
            }
          }
         
          // Get the current status for the current list of tservers
          SortedMap<TServerInstance,TabletServerStatus> currentTServers = new TreeMap<TServerInstance,TabletServerStatus>();
          for (TServerInstance entry : tserverSet.getCurrentServers()) {
            currentTServers.put(entry, tserverStatus.get(entry));
          }
         
          if (currentTServers.size() == 0) {
            eventListener.waitForEvents(TIME_TO_WAIT_BETWEEN_SCANS);
            continue;
          }
         
          // Don't move tablets to servers that are shutting down
          SortedMap<TServerInstance,TabletServerStatus> destinations = new TreeMap<TServerInstance,TabletServerStatus>(currentTServers);
          destinations.keySet().removeAll(serversToShutdown);
         
          List<Assignment> assignments = new ArrayList<Assignment>();
          List<Assignment> assigned = new ArrayList<Assignment>();
          List<TabletLocationState> assignedToDeadServers = new ArrayList<TabletLocationState>();
          Map<KeyExtent,TServerInstance> unassigned = new HashMap<KeyExtent,TServerInstance>();
         
          int[] counts = new int[TabletState.values().length];
          stats.begin();
          // Walk through the tablets in our store, and work tablets
          // towards their goal
          for (TabletLocationState tls : store) {
            if (tls == null) {
              continue;
            }
            // ignore entries for tables that do not exist in zookeeper
            if (TableManager.getInstance().getTableState(tls.extent.getTableId().toString()) == null)
              continue;
           
            // Don't overwhelm the tablet servers with work
            if (unassigned.size() + unloaded > MAX_TSERVER_WORK_CHUNK * currentTServers.size()) {
              flushChanges(destinations, assignments, assigned, assignedToDeadServers, unassigned);
              assignments.clear();
              assigned.clear();
              assignedToDeadServers.clear();
              unassigned.clear();
              unloaded = 0;
              eventListener.waitForEvents(TIME_TO_WAIT_BETWEEN_SCANS);
            }
            Text tableId = tls.extent.getTableId();
            MergeStats mergeStats = mergeStatsCache.get(tableId);
            if (mergeStats == null) {
              mergeStats = currentMerges.get(tableId);
              if (mergeStats == null) {
                mergeStats = new MergeStats(new MergeInfo());
              }
              mergeStatsCache.put(tableId, mergeStats);
            }
            TabletGoalState goal = getGoalState(tls, mergeStats.getMergeInfo());
            TServerInstance server = tls.getServer();
            TabletState state = tls.getState(currentTServers.keySet());
            stats.update(tableId, state);
            mergeStats.update(tls.extent, state, tls.chopped, !tls.walogs.isEmpty());
            sendChopRequest(mergeStats.getMergeInfo(), state, tls);
            sendSplitRequest(mergeStats.getMergeInfo(), state, tls);
           
            // Always follow through with assignments
            if (state == TabletState.ASSIGNED) {
              goal = TabletGoalState.HOSTED;
            }
View Full Code Here

Examples of org.apache.accumulo.server.master.state.MergeStats

        try {
          Map<Text,MergeStats> mergeStatsCache = new HashMap<Text,MergeStats>();
          Map<Text,MergeStats> currentMerges = new HashMap<Text,MergeStats>();
          for (MergeInfo merge : merges()) {
            if (merge.getRange() != null) {
              currentMerges.put(merge.getRange().getTableId(), new MergeStats(merge));
            }
          }
         
          // Get the current status for the current list of tservers
          SortedMap<TServerInstance,TabletServerStatus> currentTServers = new TreeMap<TServerInstance,TabletServerStatus>();
          for (TServerInstance entry : tserverSet.getCurrentServers()) {
            currentTServers.put(entry, tserverStatus.get(entry));
          }
         
          if (currentTServers.size() == 0) {
            eventListener.waitForEvents(TIME_TO_WAIT_BETWEEN_SCANS);
            continue;
          }
         
          // Don't move tablets to servers that are shutting down
          SortedMap<TServerInstance,TabletServerStatus> destinations = new TreeMap<TServerInstance,TabletServerStatus>(currentTServers);
          destinations.keySet().removeAll(serversToShutdown);
         
          List<Assignment> assignments = new ArrayList<Assignment>();
          List<Assignment> assigned = new ArrayList<Assignment>();
          List<TabletLocationState> assignedToDeadServers = new ArrayList<TabletLocationState>();
          Map<KeyExtent,TServerInstance> unassigned = new HashMap<KeyExtent,TServerInstance>();
         
          int[] counts = new int[TabletState.values().length];
          stats.begin();
          // Walk through the tablets in our store, and work tablets
          // towards their goal
          for (TabletLocationState tls : store) {
            if (tls == null) {
              continue;
            }
           
            // Don't overwhelm the tablet servers with work
            if (unassigned.size() + unloaded > MAX_TSERVER_WORK_CHUNK * currentTServers.size()) {
              flushChanges(destinations, assignments, assigned, assignedToDeadServers, unassigned);
              assignments.clear();
              assigned.clear();
              assignedToDeadServers.clear();
              unassigned.clear();
              unloaded = 0;
              eventListener.waitForEvents(TIME_TO_WAIT_BETWEEN_SCANS);
            }
            Text tableId = tls.extent.getTableId();
            MergeStats mergeStats = mergeStatsCache.get(tableId);
            if (mergeStats == null) {
              mergeStats = currentMerges.get(tableId);
              if (mergeStats == null) {
                mergeStats = new MergeStats(getMergeInfo(tableId));
              }
              mergeStatsCache.put(tableId, mergeStats);
            }
            TabletGoalState goal = getGoalState(tls, mergeStats.getMergeInfo());
            TServerInstance server = tls.getServer();
            TabletState state = tls.getState(currentTServers.keySet());
            stats.update(tableId, state);
            mergeStats.update(tls.extent, state, tls.chopped, !tls.walogs.isEmpty());
            sendChopRequest(mergeStats.getMergeInfo(), state, tls);
            sendSplitRequest(mergeStats.getMergeInfo(), state, tls);
           
            // Always follow through with assignments
            if (state == TabletState.ASSIGNED) {
              goal = TabletGoalState.HOSTED;
            }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.