Package plan_runner.storage

Examples of plan_runner.storage.TupleStorage


    super(cp, allCompNames, hierarchyPosition, conf);
   
    _firstEmitterIndex = String.valueOf(allCompNames.indexOf(firstEmitter.getName()));
    _secondEmitterIndex = String.valueOf(allCompNames.indexOf(secondEmitter.getName()));

    _firstRelationStorage = new TupleStorage();
    _secondRelationStorage = new TupleStorage();
   
    _operatorChain = cp.getChainOperator();
    _fullHashList = cp.getFullHashList();

    _aggBatchOutputMillis = cp.getBatchOutputMillis();
View Full Code Here


  private void processNonLastTuple(String inputComponentIndex, List<String> tuple,
      String inputTupleHash, Tuple stormTupleRcv, boolean isLastInBatch) {

    boolean isFromFirstEmitter = false;
    String inputTupleString = MyUtilities.tupleToString(tuple, getConf());   
    TupleStorage affectedStorage, oppositeStorage;
    List<Index> affectedIndexes, oppositeIndexes;

    if (_firstEmitterIndex.equals(inputComponentIndex)) {
      // R update
      isFromFirstEmitter = true;
      affectedStorage = _firstRelationStorage;
      oppositeStorage = _secondRelationStorage;
      affectedIndexes = _firstRelationIndexes;
      oppositeIndexes = _secondRelationIndexes;
    } else if (_secondEmitterIndex.equals(inputComponentIndex)) {
      // S update
      isFromFirstEmitter = false;
      affectedStorage = _secondRelationStorage;
      oppositeStorage = _firstRelationStorage;
      affectedIndexes = _secondRelationIndexes;
      oppositeIndexes = _firstRelationIndexes;
    } else
      throw new RuntimeException("InputComponentName " + inputComponentIndex
          + " doesn't match neither " + _firstEmitterIndex + " nor "
          + _secondEmitterIndex + ".");


    // add the stormTuple to the specific storage
    if (MyUtilities.isStoreTimestamp(getConf(), getHierarchyPosition())) {
      final long incomingTimestamp = stormTupleRcv.getLongByField(StormComponent.TIMESTAMP);
      inputTupleString = incomingTimestamp + SystemParameters.STORE_TIMESTAMP_DELIMITER
          + inputTupleString;
    }
    final int row_id = affectedStorage.insert(inputTupleString);

    List<String> valuesToApplyOnIndex = null;

    if (_existIndexes)
      valuesToApplyOnIndex = updateIndexes(inputComponentIndex, tuple, affectedIndexes, row_id);
View Full Code Here

  // join-specific code
  protected void performJoin(Tuple stormTupleRcv, List<String> tuple, String inputTupleHash,
      boolean isFromFirstEmitter, List<Index> oppositeIndexes,
      List<String> valuesToApplyOnIndex, TupleStorage oppositeStorage, boolean isLastInBatch) {

    final TupleStorage tuplesToJoin = new TupleStorage();
    selectTupleToJoin(oppositeStorage, oppositeIndexes, isFromFirstEmitter,
        valuesToApplyOnIndex, tuplesToJoin);
    join(stormTupleRcv, tuple, isFromFirstEmitter, tuplesToJoin, isLastInBatch);
  }
View Full Code Here

    }
    if (getHierarchyPosition() == FINAL_COMPONENT && (!MyUtilities.isAckEveryTuple(conf)))
      killer.registerComponent(this, parallelism);
    if (cp.getPrintOut() && _operatorChain.isBlocking())
      currentBolt.allGrouping(killer.getID(), SystemParameters.DUMP_RESULTS_STREAM);
    _firstRelationStorage = new TupleStorage();
    _secondRelationStorage = new TupleStorage();
    if (_joinPredicate != null) {
      createIndexes();
      _existIndexes = true;
    } else
      _existIndexes = false;
View Full Code Here

  }

  protected void performJoin(Tuple stormTupleRcv, List<String> tuple, String inputTupleHash,
      boolean isFromFirstEmitter, List<Index> oppositeIndexes,
      List<String> valuesToApplyOnIndex, TupleStorage oppositeStorage, boolean isLastInBatch) {
    final TupleStorage tuplesToJoin = new TupleStorage();
    selectTupleToJoin(oppositeStorage, oppositeIndexes, isFromFirstEmitter,
        valuesToApplyOnIndex, tuplesToJoin);
    join(stormTupleRcv, tuple, isFromFirstEmitter, tuplesToJoin, isLastInBatch);
  }
View Full Code Here

  private void processNonLastTuple(String inputComponentIndex, String inputTupleString, //
      List<String> tuple, // these two are the same
      String inputTupleHash, Tuple stormTupleRcv, boolean isLastInBatch) {
    boolean isFromFirstEmitter = false;
    TupleStorage affectedStorage, oppositeStorage;
    List<Index> affectedIndexes, oppositeIndexes;
    if (_firstEmitterIndex.equals(inputComponentIndex)) {
      // R update
      isFromFirstEmitter = true;
      affectedStorage = _firstRelationStorage;
      oppositeStorage = _secondRelationStorage;
      affectedIndexes = _firstRelationIndexes;
      oppositeIndexes = _secondRelationIndexes;
      sendToStatisticsCollector(tuple, 0);
    } else if (_secondEmitterIndex.equals(inputComponentIndex)) {
      // S update
      isFromFirstEmitter = false;
      affectedStorage = _secondRelationStorage;
      oppositeStorage = _firstRelationStorage;
      affectedIndexes = _secondRelationIndexes;
      oppositeIndexes = _firstRelationIndexes;
      sendToStatisticsCollector(tuple, 1);
    } else
      throw new RuntimeException("InputComponentName " + inputComponentIndex
          + " doesn't match neither " + _firstEmitterIndex + " nor "
          + _secondEmitterIndex + ".");
    // add the stormTuple to the specific storage
    if (MyUtilities.isStoreTimestamp(getConf(), getHierarchyPosition())) {
      final long incomingTimestamp = stormTupleRcv.getLongByField(StormComponent.TIMESTAMP);
      inputTupleString = incomingTimestamp + SystemParameters.STORE_TIMESTAMP_DELIMITER
          + inputTupleString;
    }
    final int row_id = affectedStorage.insert(inputTupleString);
    List<String> valuesToApplyOnIndex = null;
    if (_existIndexes)
      valuesToApplyOnIndex = updateIndexes(inputComponentIndex, tuple, affectedIndexes,
          row_id);
    performJoin(stormTupleRcv, tuple, inputTupleHash, isFromFirstEmitter, oppositeIndexes,
View Full Code Here

    constructStorageAndIndexes();

  }

  protected void constructStorageAndIndexes() {
    _firstRelationStorage = new TupleStorage();
    _firstRelationStorageNewEpoch = new TupleStorage();
    _secondRelationStorage = new TupleStorage();
    _secondRelationStorageNewEpoch = new TupleStorage();
    _firstTaggedRelationStorage = new TupleStorage();
    _firstTaggedRelationStorageNewEpoch = new TupleStorage();
    _secondTaggedRelationStorage = new TupleStorage();
    _secondTaggedRelationStorageNewEpoch = new TupleStorage();
    if (_joinPredicate != null) {
      createIndexes(1, true, false);
      createIndexes(1, false, false);
      createIndexes(2, true, false);
      createIndexes(2, false, false);
View Full Code Here

    // Now the add the epoch's tagged tuples
    // ********************************
    // add T1' to T1
    addTaggedTuples(_firstTaggedRelationStorageNewEpoch, _firstTaggedRelationStorage,
        _firstTaggedRelationIndexes, _firstEmitterIndex);
    _firstTaggedRelationStorageNewEpoch = new TupleStorage();
    createIndexes(1, true, true);
    // add T2' to T2
    addTaggedTuples(_secondTaggedRelationStorageNewEpoch, _secondTaggedRelationStorage,
        _secondTaggedRelationIndexes, _secondEmitterIndex);
    _secondTaggedRelationStorageNewEpoch = new TupleStorage();
    createIndexes(2, true, true);
  }
View Full Code Here

  /* Perform discards on the relationNumber dimension 1=first 2=second */
  protected void performDiscards(int relationNumber) {
    final PredicateCreateIndexesVisitor visitor = new PredicateCreateIndexesVisitor();
    _joinPredicate.accept(visitor);
    final TupleStorage keepStorage = new TupleStorage();
    ArrayList<Index> keepIndexes;
    String discardingEmitterIndex;
    int discardingParts, discardingIndex;

    TupleStorage discardingTupleStorage, discardingTaggedRelationStorage;
    if (relationNumber == 1) {
      discardingTupleStorage = _firstRelationStorage;
      discardingTaggedRelationStorage = _firstTaggedRelationStorage;
      keepIndexes = new ArrayList<Index>(visitor._firstRelationIndexes);
      discardingEmitterIndex = _firstEmitterIndex;
      discardingParts = _currentAction.getDiscardRowSplits();
      discardingIndex = _currentAction.getRowKeptPieceIndexByNewId(_renamedIDindex);
    } else {
      discardingTupleStorage = _secondRelationStorage;
      discardingTaggedRelationStorage = _secondTaggedRelationStorage;
      keepIndexes = new ArrayList<Index>(visitor._secondRelationIndexes);
      discardingEmitterIndex = _secondEmitterIndex;
      discardingParts = _currentAction.getDiscardColumnSplits();
      discardingIndex = _currentAction.getColumnKeptPieceIndexByNewId(_renamedIDindex);
    }

    final int[] hash = new int[discardingTaggedRelationStorage.size()
        + discardingTupleStorage.size()];
    final int[] addresses = new int[discardingTaggedRelationStorage.size()
        + discardingTupleStorage.size()];

    final BitVector isTagged = new BitVector(discardingTaggedRelationStorage.size()
        + discardingTupleStorage.size());
    isTagged.set(0, discardingTaggedRelationStorage.size());

    TupleStorage.preProcess(discardingTaggedRelationStorage, discardingTupleStorage, hash,
        addresses);

    final int bounds[] = Discard.keep(hash, addresses, isTagged, discardingParts,
        discardingIndex);

    for (int i = bounds[0]; i <= bounds[1]; i++) {
      final int address = addresses[i];
      final boolean isTaggedTuple = isTagged.get(i);
      String tupleString = "";
      if (isTaggedTuple)
        tupleString = discardingTaggedRelationStorage.get(address);
      else
        tupleString = discardingTupleStorage.get(address);
      final int row_id = keepStorage.insert(tupleString);
      if (_existIndexes) {
        if (MyUtilities.isStoreTimestamp(getConf(), getHierarchyPosition())) {
          // timestamp has to be removed
          final String parts[] = tupleString.split("\\@");
          tupleString = new String(parts[1]);
        }
        updateIndexes(discardingEmitterIndex,
            MyUtilities.stringToTuple(tupleString, getConf()), keepIndexes, row_id);
      }
    }
    if (relationNumber == 1) {
      _firstTaggedRelationStorage = keepStorage;
      _firstTaggedRelationIndexes = keepIndexes;
      _firstRelationStorage = _firstRelationStorageNewEpoch;
      _firstRelationIndexes = _firstRelationIndexesNewEpoch;
      _firstRelationStorageNewEpoch = new TupleStorage();
      createIndexes(1, false, true);
    } else {
      _secondTaggedRelationStorage = keepStorage;
      _secondTaggedRelationIndexes = keepIndexes;
      _secondRelationStorage = _secondRelationStorageNewEpoch;
      _secondRelationIndexes = _secondRelationIndexesNewEpoch;
      _secondRelationStorageNewEpoch = new TupleStorage();
      createIndexes(2, false, true);
    }
  }
View Full Code Here

    }
  }

  protected void performExchanges(int relationNumber) {

    TupleStorage exchangingRelationStorage;
    TupleStorage exchangingTaggedRelationStorage;
    List<Index> exchangingIndexes;

    if (relationNumber == 1) {
      exchangingIndexes = _firstTaggedRelationIndexes;
      exchangingTaggedRelationStorage = _firstTaggedRelationStorage;
      exchangingRelationStorage = _firstRelationStorage;
      _migratingRelationStorage = _firstRelationStorage;
      _migratingTaggedRelationStorage = _firstTaggedRelationStorage;
      _migratingTaggedRelationSize = _firstTaggedRelationStorage.size();
      _migratingRelationCursor = 0;
      _migratingTaggedRelationCursor = 0;
      _migratingRelationIndex = _firstEmitterIndex;
    } else {
      exchangingIndexes = _secondTaggedRelationIndexes;
      exchangingTaggedRelationStorage = _secondTaggedRelationStorage;
      exchangingRelationStorage = _secondRelationStorage;
      _migratingRelationStorage = _secondRelationStorage;
      _migratingTaggedRelationStorage = _secondTaggedRelationStorage;
      _migratingTaggedRelationSize = _secondTaggedRelationStorage.size();
      _migratingRelationCursor = 0;
      _migratingTaggedRelationCursor = 0;
      _migratingRelationIndex = _secondEmitterIndex;
    }

    if (exchangingRelationStorage.size() <= migrationBufferSize)
      emitBulk(exchangingRelationStorage, _migratingRelationIndex, 0,
          exchangingRelationStorage.size() - 1);
    else {
      _isMigratingRelationStorage = true;
      _migratingRelationCursor += migrationBufferSize;
      emitBulk(_migratingRelationStorage, _migratingRelationIndex, 0,
          _migratingRelationCursor - 1);
    }
    if (_migratingTaggedRelationSize <= migrationBufferSize)
      emitBulk(exchangingTaggedRelationStorage, _migratingRelationIndex, 0,
          _migratingTaggedRelationSize - 1);
    else {
      _isMigratingTaggedRelationStorage = true;
      _migratingTaggedRelationCursor += migrationBufferSize;
      emitBulk(_migratingTaggedRelationStorage, _migratingRelationIndex, 0,
          _migratingTaggedRelationCursor - 1);
    }
    // Restart everything to be tagged
    addTaggedTuples(exchangingRelationStorage, exchangingTaggedRelationStorage,
        exchangingIndexes, _migratingRelationIndex);

    if (_isMigratingRelationStorage) // if it was bigger than buffer size, and still has more data
      clean(_migratingRelationStorage, 0, _migratingRelationCursor - 1); // remove that data
    if (relationNumber == 1) {
      _firstRelationStorage = _firstRelationStorageNewEpoch;
      _firstRelationIndexes = _firstRelationIndexesNewEpoch;
      _firstRelationStorageNewEpoch = new TupleStorage();
      createIndexes(1, false, true);// reinitialize untagged
    } else {
      _secondRelationStorage = _secondRelationStorageNewEpoch;
      _secondRelationIndexes = _secondRelationIndexesNewEpoch;
      _secondRelationStorageNewEpoch = new TupleStorage();
      createIndexes(2, false, true);// reinitialize untagged
    }

    if (_isMigratingRelationStorage || _isMigratingTaggedRelationStorage) {
      final Values tplSend = new Values("N/A", MyUtilities.stringToTuple(
View Full Code Here

TOP

Related Classes of plan_runner.storage.TupleStorage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.