Package de.lmu.ifi.dbs.elki.database.datastore

Examples of de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore


        }
        Rk = Rnew;
      }
    }

    WritableDoubleDataStore sparsity = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_STATIC);
    // calculate the sparsity coefficient
    for(Vector<IntIntPair> sub : Rk) {
      DBIDs ids = computeSubspace(sub, ranges);
      final double sparsityC = sparsity(ids.size(), size, k);

      if(sparsityC < 0) {
        for(DBID id : ids) {
          double prev = sparsity.doubleValue(id);
          if(Double.isNaN(prev) || sparsityC < prev) {
            sparsity.putDouble(id, sparsityC);
          }
        }
      }
    }
    DoubleMinMax minmax = new DoubleMinMax();
    for(DBID id : relation.iterDBIDs()) {
      double val = sparsity.doubleValue(id);
      if(Double.isNaN(val)) {
        sparsity.putDouble(id, 0.0);
        val = 0.0;
      }
      minmax.put(val);
    }
    Relation<Double> scoreResult = new MaterializedRelation<Double>("AggarwalYuNaive", "aggarwal-yu-outlier", TypeUtil.DOUBLE, sparsity, relation.getDBIDs());
View Full Code Here


    DistanceQuery<V, D> distFunc = relation.getDatabase().getDistanceQuery(relation, distanceFunction);
    Collection<V> refPoints = refp.getReferencePoints(relation);

    DBIDs ids = relation.getDBIDs();
    // storage of distance/score values.
    WritableDoubleDataStore rbod_score = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_STATIC | DataStoreFactory.HINT_HOT);

    // Compute density estimation:
    {
      // compute density for one reference point, to initialize the first
      // density
      // value for each object, then update
      final Iterator<V> iter = refPoints.iterator();
      if(!iter.hasNext()) {
        throw new AbortException("Cannot compute ROS without reference points!");
      }
      V firstRef = iter.next();
      // compute distance vector for the first reference point
      List<DistanceResultPair<D>> firstReferenceDists = computeDistanceVector(firstRef, relation, distFunc);
      for(int l = 0; l < firstReferenceDists.size(); l++) {
        double density = computeDensity(firstReferenceDists, l);
        // Initial value
        rbod_score.putDouble(firstReferenceDists.get(l).getDBID(), density);
      }
      // compute density values for all remaining reference points
      while(iter.hasNext()) {
        V refPoint = iter.next();
        List<DistanceResultPair<D>> referenceDists = computeDistanceVector(refPoint, relation, distFunc);
        // compute density value for each object
        for(int l = 0; l < referenceDists.size(); l++) {
          double density = computeDensity(referenceDists, l);
          // Update minimum
          if(density < rbod_score.doubleValue(referenceDists.get(l).getDBID())) {
            rbod_score.putDouble(referenceDists.get(l).getDBID(), density);
          }
        }
      }
    }
    // compute maximum density
    double maxDensity = 0.0;
    for(DBID id : relation.iterDBIDs()) {
      double dens = rbod_score.doubleValue(id);
      if(dens > maxDensity) {
        maxDensity = dens;
      }
    }
    // compute ROS
    for(DBID id : relation.iterDBIDs()) {
      double score = 1 - (rbod_score.doubleValue(id) / maxDensity);
      rbod_score.putDouble(id, score);
    }

    // adds reference points to the result. header information for the
    // visualizer to find the reference points in the result
    ReferencePointsResult<V> refp = new ReferencePointsResult<V>("Reference points", "reference-points", refPoints);
View Full Code Here

    KNNQuery<O, D> knnQuery = database.getKNNQuery(distFunc, DatabaseQuery.HINT_OPTIMIZED_ONLY);

    // maximum number of objects in the D-neighborhood of an outlier
    int m = (int) ((distFunc.getRelation().size()) * (1 - p));

    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(distFunc.getRelation().getDBIDs(), DataStoreFactory.HINT_STATIC);
    if(logger.isVerbose()) {
      logger.verbose("computing outlier flag");
    }

    FiniteProgress progressOFlags = logger.isVerbose() ? new FiniteProgress("DBOutlier for objects", distFunc.getRelation().size(), logger) : null;
    int counter = 0;
    // if index exists, kNN query. if the distance to the mth nearest neighbor
    // is more than d -> object is outlier
    if(knnQuery != null) {
      for(DBID id : distFunc.getRelation().iterDBIDs()) {
        counter++;
        final KNNResult<D> knns = knnQuery.getKNNForDBID(id, m);
        if(logger.isDebugging()) {
          logger.debugFine("distance to mth nearest neighbour" + knns.toString());
        }
        if(knns.get(Math.min(m, knns.size()) - 1).getDistance().compareTo(neighborhoodSize) <= 0) {
          // flag as outlier
          scores.putDouble(id, 1.0);
        }
        else {
          // flag as no outlier
          scores.putDouble(id, 0.0);
        }
      }
      if(progressOFlags != null) {
        progressOFlags.setProcessed(counter, logger);
      }
    }
    else {
      // range query for each object. stop if m objects are found
      for(DBID id : distFunc.getRelation().iterDBIDs()) {
        counter++;
        Iterator<DBID> iterator = distFunc.getRelation().iterDBIDs();
        int count = 0;
        while(iterator.hasNext() && count < m) {
          DBID currentID = iterator.next();
          D currentDistance = distFunc.distance(id, currentID);

          if(currentDistance.compareTo(neighborhoodSize) <= 0) {
            count++;
          }
        }

        if(count < m) {
          // flag as outlier
          scores.putDouble(id, 1.0);
        }
        else {
          // flag as no outlier
          scores.putDouble(id, 0.0);
        }
      }

      if(progressOFlags != null) {
        progressOFlags.setProcessed(counter, logger);
View Full Code Here

  protected DataStore<Double> computeOutlierScores(Database database, Relation<O> relation, D d) {
    DistanceQuery<O, D> distFunc = database.getDistanceQuery(relation, getDistanceFunction());
    RangeQuery<O, D> rangeQuery = database.getRangeQuery(distFunc);
    final double size = distFunc.getRelation().size();

    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(distFunc.getRelation().getDBIDs(), DataStoreFactory.HINT_STATIC);
    // TODO: use bulk when implemented.
    for(DBID id : distFunc.getRelation().iterDBIDs()) {
      // compute percentage of neighbors in the given neighborhood with size d
      double n = (rangeQuery.getRangeForDBID(id, d).size()) / size;
      scores.putDouble(id, 1.0 - n);
    }
    return scores;
  }
View Full Code Here

      }
      ArrayDBIDs lrd_ids = DBIDUtil.ensureArray(DBIDUtil.union(insertions, updates2));
      List<List<DistanceResultPair<D>>> reachDistRKNNs = lofResult.getRkNNReach().getRKNNForBulkDBIDs(lrd_ids, k);
      ArrayDBIDs affected_lrd_id_candidates = mergeIDs(reachDistRKNNs, lrd_ids);
      ArrayModifiableDBIDs affected_lrd_ids = DBIDUtil.newArray(affected_lrd_id_candidates.size());
      WritableDoubleDataStore new_lrds = computeLRDs(affected_lrd_id_candidates, lofResult.getKNNReach());
      for(DBID id : affected_lrd_id_candidates) {
        double new_lrd = new_lrds.doubleValue(id);
        double old_lrd = lofResult.getLrds().doubleValue(id);
        if(Double.isNaN(old_lrd) || old_lrd != new_lrd) {
          lofResult.getLrds().putDouble(id, new_lrd);
          affected_lrd_ids.add(id);
        }
View Full Code Here

      }
      ArrayDBIDs lrd_ids = DBIDUtil.ensureArray(updates2);
      List<List<DistanceResultPair<D>>> reachDistRKNNs = lofResult.getRkNNReach().getRKNNForBulkDBIDs(lrd_ids, k);
      ArrayDBIDs affected_lrd_id_candidates = mergeIDs(reachDistRKNNs, lrd_ids);
      ArrayModifiableDBIDs affected_lrd_ids = DBIDUtil.newArray(affected_lrd_id_candidates.size());
      WritableDoubleDataStore new_lrds = computeLRDs(affected_lrd_id_candidates, lofResult.getKNNReach());
      for(DBID id : affected_lrd_id_candidates) {
        double new_lrd = new_lrds.doubleValue(id);
        double old_lrd = lofResult.getLrds().doubleValue(id);
        if(old_lrd != new_lrd) {
          lofResult.getLrds().putDouble(id, new_lrd);
          affected_lrd_ids.add(id);
        }
View Full Code Here

     * @param ids the ids of the lofs to be recomputed
     * @param lofResult the result of the former LOF run
     */
    private void recomputeLOFs(DBIDs ids, LOFResult<O, D> lofResult) {
      Pair<WritableDoubleDataStore, DoubleMinMax> lofsAndMax = computeLOFs(ids, lofResult.getLrds(), lofResult.getKNNRefer());
      WritableDoubleDataStore new_lofs = lofsAndMax.getFirst();
      for(DBID id : ids) {
        lofResult.getLofs().putDouble(id, new_lofs.doubleValue(id));
      }
      // track the maximum value for normalization.
      DoubleMinMax new_lofminmax = lofsAndMax.getSecond();

      // Actualize meta info
View Full Code Here

   * @param vecs Vector relation
   * @param labels Label relation
   * @return Outlier result
   */
  public OutlierResult run(Relation<Model> models, Relation<NumberVector<?, ?>> vecs, Relation<?> labels) {
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(models.getDBIDs(), DataStoreFactory.HINT_HOT);

    // Adjustment constant
    final double minscore = expect / (expect + 1);
   
    HashSet<GeneratorSingleCluster> generators = new HashSet<GeneratorSingleCluster>();
    for(DBID id : models.iterDBIDs()) {
      Model model = models.get(id);
      if(model instanceof GeneratorSingleCluster) {
        generators.add((GeneratorSingleCluster) model);
      }
    }
    if(generators.size() == 0) {
      logger.warning("No generator models found for dataset - all points will be considered outliers.");
    }

    for(DBID id : models.iterDBIDs()) {
      double score = 0.0;
      // Convert to a math vector
      Vector v = vecs.get(id).getColumnVector();
      for(GeneratorSingleCluster gen : generators) {
        Vector tv = v;
        // Transform backwards
        if(gen.getTransformation() != null) {
          tv = gen.getTransformation().applyInverse(v);
        }
        final int dim = tv.getDimensionality();
        double lensq = 0.0;
        int norm = 0;
        for(int i = 0; i < dim; i++) {
          Distribution dist = gen.getDistribution(i);
          if(dist instanceof NormalDistribution) {
            NormalDistribution d = (NormalDistribution) dist;
            double delta = (tv.get(i) - d.getMean()) / d.getStddev();
            lensq += delta * delta;
            norm += 1;
          }
        }
        if(norm > 0) {
          // The squared distances are ChiSquared distributed
          score = Math.max(score, 1 - ChiSquaredDistribution.cdf(lensq, norm));
        }
      }
      // score inversion.
      score = expect / (expect + score);
      // adjust to 0 to 1 range:
      score = (score - minscore) / (1 - minscore);
      scores.putDouble(id, score);
    }
    Relation<Double> scoreres = new MaterializedRelation<Double>("Model outlier scores", "model-outlier", TypeUtil.DOUBLE, scores, models.getDBIDs());
    OutlierScoreMeta meta = new ProbabilisticOutlierScore(0., 1.);
    return new OutlierResult(meta, scoreres);
  }
View Full Code Here

   *
   * @param relation Relation
   * @return Result
   */
  public OutlierResult run(Relation<?> relation) throws IllegalStateException {
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT);
    for(DBID id : relation.iterDBIDs()) {
      scores.putDouble(id, 0.0);
    }
    Relation<Double> scoreres = new MaterializedRelation<Double>("Trivial no-outlier score", "no-outlier", TypeUtil.DOUBLE, scores, relation.getDBIDs());
    OutlierScoreMeta meta = new ProbabilisticOutlierScore();
    return new OutlierResult(meta, scoreres);
  }
View Full Code Here

   *
   * @param relation Relation to process.
   * @return Result
   */
  public OutlierResult run(Relation<?> relation) {
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT);
    for(DBID id : relation.iterDBIDs()) {
      String label = relation.get(id).toString();
      final double score;
      if (pattern.matcher(label).matches()) {
        score = 1.0;
      } else {
        score = 0.0;
      }
      scores.putDouble(id, score);
    }
    Relation<Double> scoreres = new MaterializedRelation<Double>("By label outlier scores", "label-outlier", TypeUtil.DOUBLE, scores, relation.getDBIDs());
    OutlierScoreMeta meta = new ProbabilisticOutlierScore();
    return new OutlierResult(meta, scoreres);
  }
View Full Code Here

TOP

Related Classes of de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.