Package edu.brown.hstore.estimators

Examples of edu.brown.hstore.estimators.EstimatorState


                item_id,                // ITEM_IDS
                supware,                // SUPPLY W_IDS
                quantity                // QUANTITIES
            };
           
            EstimatorState state = this.estimator.startTransaction(txn_id, catalog_proc, args);
            assertNotNull(state);
            assertEquals(txn_id, state.getTransactionId().longValue());
//            System.err.printf("W_ID=%d / S_W_ID=%s\n", w_id, remote_w_id);
           
            // Make sure that it identifies that we are a distributed transaction and
            // that we expect to touch both the W_ID partition and the REMOTE_W_ID partition
            Estimate est = state.getInitialEstimate();
            assertNotNull(est);
            PartitionSet partitions = est.getTouchedPartitions(thresholds);
            assertEquals(2, partitions.size());
            for (int expected : new int[]{ w_id, remote_w_id }) {
                expected = hasher.hash(expected);
                assertTrue(Integer.toString(expected) + "->" + partitions, partitions.contains(expected));
            } // FOR
            assertEquals(hasher.hash(w_id), state.getBasePartition());
            txn_id++;
        } // FOR
    }
View Full Code Here


       
        LocalTransaction ts = lastTxn.get();
        assertNotNull(ts);
        assertFalse(ts.isPredictSinglePartition());
       
        EstimatorState estState = ts.getEstimatorState();
        assertNotNull(estState);
        assertEquals(MarkovEstimatorState.class, estState.getClass());
        assertTrue(estState.isInitialized());
       
        Estimate est = estState.getInitialEstimate();
        assertNotNull(est);
        assertEquals(MarkovEstimate.class, est.getClass());
       
       
    }
View Full Code Here

                                    RpcCallback<ClientResponseImpl> client_callback) {
        final int procId = catalog_proc.getId();
        boolean predict_abortable = (hstore_conf.site.exec_no_undo_logging_all == false);
        boolean predict_readOnly = this.isReadOnly[procId];
        PartitionSet predict_partitions = null;
        EstimatorState t_state = null;
       
        // Setup TransactionProfiler
        if (hstore_conf.site.txn_profiling) {
            this.setupTransactionProfiler(ts, this.isSysProc[procId]);
        }
       
        // -------------------------------
        // SYSTEM PROCEDURES
        // -------------------------------
        if (this.isSysProc[procId]) {
            // Sysprocs can be either all partitions or single-partitioned
            // TODO: It would be nice if the client could pass us a hint when loading the tables
            // It would be just for the loading, and not regular transactions
            if (catalog_proc.getSinglepartition() && catalog_proc.getEverysite() == false) {
                predict_partitions = catalogContext.getPartitionSetSingleton(base_partition);
            } else {
                predict_partitions = catalogContext.getAllPartitionIds();
            }
        }
        // -------------------------------
        // MAPREDUCE TRANSACTIONS
        // -------------------------------
        else if (this.isMapReduce[procId]) {
            // MapReduceTransactions always need all partitions
            if (debug.val)
                LOG.debug(String.format("New request is for MapReduce %s, so it has to be " +
                      "multi-partitioned [clientHandle=%d]",
                          catalog_proc.getName(), ts.getClientHandle()));
            predict_partitions = catalogContext.getAllPartitionIds();
        }
        // -------------------------------
        // VOLTDB @PROCINFO
        // -------------------------------
        else if (hstore_conf.site.exec_voltdb_procinfo) {
            if (debug.val)
                LOG.debug(String.format("Using the catalog information to determine whether the %s transaction " +
                      "is single-partitioned [clientHandle=%d, singleP=%s]",
                          catalog_proc.getName(), ts.getClientHandle(), catalog_proc.getSinglepartition()));
            if (catalog_proc.getSinglepartition()) {
                predict_partitions = catalogContext.getPartitionSetSingleton(base_partition);
            } else {
                predict_partitions = catalogContext.getAllPartitionIds();
            }
        }
        // -------------------------------
        // FORCE DISTRIBUTED
        // -------------------------------
        else if (hstore_conf.site.exec_force_allpartitions) {
            predict_partitions = catalogContext.getAllPartitionIds();
        }
        // -------------------------------
        // TRANSACTION ESTIMATORS
        // -------------------------------
        else if (hstore_conf.site.markov_enable || hstore_conf.site.markov_fixed) {
            // Grab the TransactionEstimator for the destination partition and figure out whether
            // this mofo is likely to be single-partition or not. Anything that we can't estimate
            // will just have to be multi-partitioned. This includes sysprocs
            TransactionEstimator t_estimator = this.t_estimators[base_partition];
            if (t_estimator == null) {
                t_estimator = this.hstore_site.getPartitionExecutor(base_partition).getTransactionEstimator();
                this.t_estimators[base_partition] = t_estimator;
            }
           
            try {
                if (hstore_conf.site.txn_profiling && ts.profiler != null) ts.profiler.startInitEstimation();
                if (t_estimator != null) {
                    if (debug.val)
                        LOG.debug(String.format("%s - Using %s to populate txn properties [clientHandle=%d]",
                                  TransactionUtil.formatTxnName(catalog_proc, txn_id),
                                  t_estimator.getClass().getSimpleName(), client_handle));
                    t_state = t_estimator.startTransaction(txn_id, base_partition, catalog_proc, params.toArray());
                }
               
                // If there is no EstimatorState, then there is nothing we can do
                // It has to be executed as multi-partitioned
                if (t_state == null) {
                    if (debug.val) {
                        LOG.debug(String.format("%s - No %s was returned. Using default estimate.",
                                  TransactionUtil.formatTxnName(catalog_proc, txn_id),
                                  EstimatorState.class.getSimpleName()));
                    }
                }
                // We have a EstimatorState handle, so let's see what it says...
                else {
                    if (trace.val)
                        LOG.trace("\n" + StringBoxUtil.box(t_state.toString()));
                    Estimate t_estimate = t_state.getInitialEstimate();
                   
                    // Bah! We didn't get back a Estimation for some reason...
                    if (t_estimate == null) {
                        if (debug.val)
                            LOG.debug(String.format("%s - No %s handle was return. Using default estimate.",
                                      TransactionUtil.formatTxnName(catalog_proc, txn_id),
                                      Estimate.class.getSimpleName()));
                    }
                    // Invalid Estimation. Stick with defaults
                    else if (t_estimate.isValid() == false) {
                        if (debug.val)
                            LOG.debug(String.format("%s - %s is marked as invalid. Using default estimate.\n%s",
                                      TransactionUtil.formatTxnName(catalog_proc, txn_id),
                                      t_estimate.getClass().getSimpleName(), t_estimate));
                    }   
                    // Use Estimation to determine things
                    else {
                        if (debug.val) {
                            LOG.debug(String.format("%s - Using %s to determine if txn is single-partitioned",
                                      TransactionUtil.formatTxnName(catalog_proc, txn_id),
                                      t_estimate.getClass().getSimpleName()));
                            LOG.trace(String.format("%s %s:\n%s",
                                      TransactionUtil.formatTxnName(catalog_proc, txn_id),
                                      t_estimate.getClass().getSimpleName(), t_estimate));
                        }
                        predict_partitions = t_estimate.getTouchedPartitions(this.thresholds);
                        predict_readOnly = t_estimate.isReadOnlyAllPartitions(this.thresholds);
                        predict_abortable = (predict_partitions.size() == 1 ||
                                             predict_readOnly == false ||
                                             t_estimate.isAbortable(this.thresholds));
                       
                        // Check whether the TransactionEstimator *really* thinks that we should
                        // give it updates about this txn. If the flag is false, then we'll
                        // check whether the updates are enabled in the HStoreConf parameters
                        if (t_state.shouldAllowUpdates() == false) {
                            if (predict_partitions.size() == 1) {
                                if (hstore_conf.site.markov_singlep_updates == false) t_state.disableUpdates();
                            }
                            else if (hstore_conf.site.markov_dtxn_updates == false) {
                                t_state.disableUpdates();
                            }
                        }
                       
                        if (debug.val && predict_partitions.isEmpty()) {
                            LOG.warn(String.format("%s - Unexpected empty predicted %s from %s [updatesEnabled=%s]\n%s",
                                TransactionUtil.formatTxnName(catalog_proc, txn_id),
                                PartitionSet.class.getSimpleName(),
                                t_estimator.getClass().getSimpleName(),
                                t_state.isUpdatesEnabled(), t_estimate));
                            ts.setAllowEarlyPrepare(false);
//                            System.err.println("WROTE MARKOVGRAPH: " + ((MarkovEstimatorState)t_state).dumpMarkovGraph());
//                            System.err.flush();
//                            HStore.crashDB();
                        }
View Full Code Here

        // return the object back into the pool
        final Long txn_id = ts.getTransactionId();
        AbstractTransaction rm = this.inflight_txns.remove(txn_id);
        if (debug.val) LOG.debug(String.format("Deleted %s [%s / inflightRemoval:%s]", ts, status, (rm != null)));
       
        EstimatorState t_state = ts.getEstimatorState();
        if (t_state != null) {
            this.remoteTxnEstimator.destroyEstimatorState(t_state);
        }
       
        if (debug.val) {
View Full Code Here

        assert(ts.checkDeletableFlag()) :
            String.format("Trying to delete %s before it was marked as ready!", ts);
       
        // Clean-up any extra information that we may have for the txn
        TransactionEstimator t_estimator = null;
        EstimatorState t_state = ts.getEstimatorState();
        if (t_state != null) {
            t_estimator = this.executors[base_partition].getTransactionEstimator();
            assert(t_estimator != null);
        }
        if (ts.hasDependencyTracker()) {
            // HACK: Check whether there were unnecessary prefetch queries
            if (hstore_conf.site.txn_profiling && ts.profiler != null) {
                Integer cnt = this.depTrackers[base_partition].getDebugContext().getUnusedPrefetchResultCount(ts);
                if (cnt != null) ts.profiler.addPrefetchUnusedQuery(cnt.intValue());
            }
            this.depTrackers[base_partition].removeTransaction(ts);
        }
       
        // Update Transaction profiler
        // XXX: Should we include totals for mispredicted txns?
        if (hstore_conf.site.txn_profiling &&
                ts.profiler != null &&
                ts.profiler.isDisabled() == false &&
                status != Status.ABORT_MISPREDICT) {
            ts.profiler.stopTransaction();
            if (this.txnProfilerStats != null) {
                this.txnProfilerStats.addTxnProfile(ts.getProcedure(), ts.profiler);
            }
            if (this.status_monitor != null) {
                this.status_monitor.addTxnProfile(ts.getProcedure(), ts.profiler);
            }
        }
       
        try {
            switch (status) {
                case OK:
                    if (t_estimator != null) {
                        if (trace.val)
                            LOG.trace(String.format("Telling the %s to COMMIT %s",
                                      t_estimator.getClass().getSimpleName(), ts));
                        t_estimator.commit(t_state);
                    }
                    // We always need to keep track of how many txns we process
                    // in order to check whether we are hung or not
                    if (hstore_conf.site.txn_counters || hstore_conf.site.status_kill_if_hung) {
                        TransactionCounter.COMPLETED.inc(catalog_proc);
                    }
                    break;
                case ABORT_USER:
                    if (t_estimator != null) {
                        if (trace.val) LOG.trace("Telling the TransactionEstimator to ABORT " + ts);
                        t_estimator.abort(t_state, status);
                    }
                    if (hstore_conf.site.txn_counters)
                        TransactionCounter.ABORTED.inc(catalog_proc);
                    break;
                case ABORT_MISPREDICT:
                case ABORT_RESTART:
                case ABORT_EVICTEDACCESS:
                case ABORT_SPECULATIVE:
                    if (t_estimator != null) {
                        if (trace.val) LOG.trace("Telling the TransactionEstimator to IGNORE " + ts);
                        t_estimator.abort(t_state, status);
                    }
                    if (hstore_conf.site.txn_counters) {
                        if (status == Status.ABORT_EVICTEDACCESS) {
                //if(ts.getRestartCounter()==0){
                                TransactionCounter.EVICTEDACCESS.inc(catalog_proc);
                //}
                        }
                        else if (status == Status.ABORT_SPECULATIVE) {
                            TransactionCounter.ABORT_SPECULATIVE.inc(catalog_proc);
                        }
                        else if (status == Status.ABORT_MISPREDICT) {
                            TransactionCounter.MISPREDICTED.inc(catalog_proc);
                        }
                        // Don't count restarted txns more than once
                        else if (ts.getRestartCounter() == 0) {
                            TransactionCounter.RESTARTED.inc(catalog_proc);
                        }
                    }
                    break;
                case ABORT_REJECT:
                    if (hstore_conf.site.txn_counters)
                        TransactionCounter.REJECTED.inc(catalog_proc);
                    break;
                case ABORT_UNEXPECTED:
                    if (hstore_conf.site.txn_counters)
                        TransactionCounter.ABORT_UNEXPECTED.inc(catalog_proc);
                    break;
                case ABORT_GRACEFUL:
                    if (hstore_conf.site.txn_counters)
                        TransactionCounter.ABORT_GRACEFUL.inc(catalog_proc);
                    break;
                default:
                    LOG.warn(String.format("Unexpected status %s for %s", status, ts));
            } // SWITCH
        } catch (Throwable ex) {
            LOG.error(String.format("Unexpected error when cleaning up %s transaction %s", status, ts), ex);
            // Pass...
        } finally {
            if (t_state != null && t_estimator != null) {
                assert(txn_id == t_state.getTransactionId()) :
                    String.format("Unexpected mismatch txnId in %s [%d != %d]",
                                  t_state.getClass().getSimpleName(),
                                  txn_id, t_state.getTransactionId());
                t_estimator.destroyEstimatorState(t_state);
            }
        }
       
        // Update additional transaction profiling counters
View Full Code Here

            return (false);
        }
       
        // Get the estimates for both of the txns
        // If we don't have an estimate, then we have to say that there is a conflict.
        EstimatorState dtxnState = dtxn.getEstimatorState();
        EstimatorState tsState = candidate.getEstimatorState();
        if (dtxnState == null) {
            if (debug.val)
                LOG.debug(String.format("No %s available for distributed txn %s",
                          EstimatorState.class.getSimpleName(), dtxn));
            return (true);
        }
        else if (tsState == null) {
            if (debug.val)
                LOG.debug(String.format("No %s available for candidate txn %s",
                          EstimatorState.class.getSimpleName(), candidate));
            return (true);
        }
       
        // Get the current TransactionEstimate for the DTXN and the
        // initial TransactionEstimate for the single-partition txn
        // We need to make sure that both estimates have the list of
        // queries that the transaction is going to execute
        Estimate dtxnEst = dtxnState.getLastEstimate();
        if (dtxnEst == null) {
            if (debug.val)
                LOG.warn(String.format("Unexpected null %s in the %s for %s",
                         Estimate.class.getSimpleName(), dtxnState.getClass().getSimpleName(), dtxn));
            return (true);
        }
        else if (dtxnEst.hasQueryEstimate(partitionId) == false) {
            if (debug.val)
                LOG.warn(String.format("No query list estimate is available for dtxn %s", dtxn));
            return (true);
        }
        Estimate tsEst = tsState.getInitialEstimate();
        assert(tsEst != null);
        if (tsEst.hasQueryEstimate(partitionId) == false) {
            if (debug.val)
                LOG.warn(String.format("No query list estimate is available for candidate %s", candidate));
            return (true);
View Full Code Here

        Statement catalog_stmt = this.getStatement(catalog_proc, TARGET_PREFETCH_STATEMENT);
        catalog_stmt.setPrefetchable(true);
        catalog_proc.setPrefetchable(true);
       
        final ParameterSet params = new ParameterSet(this.proc_params);
        final EstimatorState estState = new MarkovEstimatorState.Factory(catalogContext).makeObject();
        estState.addPrefetchableStatement(new CountedStatement(catalog_stmt, 0));

        // Hard-code ParameterMapping
        int mappings[][] = {
            // StmtParameter -> ProcParameter
            { 0, 1 },
        };
        List<ProcParameter> procParams = CatalogUtil.getSortedCatalogItems(catalog_proc.getParameters(), "index");
        List<StmtParameter> stmtParams = CatalogUtil.getSortedCatalogItems(catalog_stmt.getParameters(), "index");
        assertNotNull(stmtParams);
        assertEquals(catalog_stmt.getParameters().size(), mappings.length);
        for (int m[] : mappings) {
            stmtParams.get(m[0]).setProcparameter(procParams.get(m[1]));
        } // FOR

        assertNotNull(catalogContext.paramMappings);
       
        this.prefetcher = new PrefetchQueryPlanner(catalogContext, p_estimator);
        SQLStmt[] batchStmts = {new SQLStmt(catalog_stmt)};
        this.prefetcher.addPlanner(catalog_proc, estState.getPrefetchableStatements(), batchStmts);
       
        for (int i = 0; i < NUM_SITES; i++) {
            this.hstore_sites[i] = new MockHStoreSite(i, catalogContext, HStoreConf.singleton());
            this.coordinators[i] = this.hstore_sites[i].initHStoreCoordinator();
View Full Code Here

            LOG.trace(ts + " - Next BatchPlan:\n" + plan.toString());
        }
        if (needs_profiling) ts.profiler.stopExecPlanning();
       
        // Tell the TransactionEstimator that we're about to execute these mofos
        EstimatorState t_state = ts.getEstimatorState();
        if (this.localTxnEstimator != null && t_state != null && t_state.isUpdatesEnabled()) {
            if (needs_profiling) ts.profiler.startExecEstimation();
            try {
                this.localTxnEstimator.executeQueries(t_state,
                                                      planner.getStatements(),
                                                      plan.getStatementPartitions());
            } finally {
                if (needs_profiling) ts.profiler.stopExecEstimation();
            }
        } else if (t_state != null && t_state.shouldAllowUpdates()) {
            LOG.warn("Skipping estimator updates for " + ts);
        }

        // Check whether our plan was caused a mispredict
        // Doing it this way allows us to update the TransactionEstimator before we abort the txn
View Full Code Here

                    next = localTxn;
                }
                // Scheduling Policy: SHORTEST/LONGEST TIME
                else {
                    // Estimate the time that remains.
                    EstimatorState es = localTxn.getEstimatorState();
                    if (es != null) {
                        long remainingTime = es.getLastEstimate().getRemainingExecutionTime();
                        if ((this.policyType == SpecExecSchedulerPolicyType.SHORTEST && remainingTime < bestTime) ||
                            (this.policyType == SpecExecSchedulerPolicyType.LONGEST && remainingTime > bestTime)) {
                            bestTime = remainingTime;
                            next = localTxn;
                            if (debug.val)
View Full Code Here

        // Add a bunch and then set the last one to have the shortest time
        this.populateQueue(this.addedTxns, 20);
        AbstractTransaction shortest = CollectionUtil.last(this.work_queue);
        for (AbstractTransaction ts : this.work_queue) {
            final long remaining = (ts == shortest ? 10 : 1000);
            EstimatorState state = new EstimatorState(catalogContext) {
                {
                    MockEstimate est = new MockEstimate(remaining);
                    this.addEstimate(est);
                }
            };
View Full Code Here

TOP

Related Classes of edu.brown.hstore.estimators.EstimatorState

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.