Package de.lmu.ifi.dbs.elki.database.datastore

Examples of de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore


   * @param database Database to use
   * @param relation Relation to use
   * @return Result
   */
  public OutlierResult run(Database database, Relation<?> relation) {
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);

    Pattern colSep = Pattern.compile(AbstractParser.WHITESPACE_PATTERN);
    DoubleMinMax minmax = new DoubleMinMax();
    InputStream in;
    try {
      in = FileUtil.tryGzipInput(new FileInputStream(file));
      BufferedReader reader = new BufferedReader(new InputStreamReader(in));

      for(String line; (line = reader.readLine()) != null;) {
        if(line.startsWith(COMMENT)) {
          continue;
        }
        else if(line.length() > 0) {
          String[] cols = colSep.split(line);
          Integer id = null;
          double score = Double.NaN;
          for(String str : cols) {
            Matcher mi = idpattern.matcher(str);
            Matcher ms = scorepattern.matcher(str);
            final boolean mif = mi.find();
            final boolean msf = ms.find();
            if(mif && msf) {
              throw new AbortException("ID pattern and score pattern both match value: " + str);
            }
            if(mif) {
              if(id != null) {
                throw new AbortException("ID pattern matched twice: previous value " + id + " second value: " + str);
              }
              id = Integer.parseInt(str.substring(mi.end()));
            }
            if(msf) {
              if(!Double.isNaN(score)) {
                throw new AbortException("Score pattern matched twice: previous value " + score + " second value: " + str);
              }
              score = Double.parseDouble(str.substring(ms.end()));
            }
          }
          if(id != null && !Double.isNaN(score)) {
            scores.putDouble(DBIDUtil.importInteger(id), score);
            minmax.put(score);
          }
          else if(id == null && Double.isNaN(score)) {
            logger.warning("Line did not match either ID nor score nor comment: " + line);
          }
          else {
            throw new AbortException("Line matched only ID or only SCORE patterns: " + line);
          }
        }
      }
    }
    catch(IOException e) {
      throw new AbortException("Could not load outlier scores: " + e.getMessage() + " when loading " + file, e);
    }

    OutlierScoreMeta meta;
    if(inverted) {
      meta = new InvertedOutlierScoreMeta(minmax.getMin(), minmax.getMax());
    }
    else {
      meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax());
    }
    Relation<Double> scoresult = new MaterializedRelation<Double>("External Outlier", "external-outlier", TypeUtil.DOUBLE, scores, relation.getDBIDs());
    OutlierResult or = new OutlierResult(meta, scoresult);

    // Apply scaling
    if(scaling instanceof OutlierScalingFunction) {
      ((OutlierScalingFunction) scaling).prepare(or);
    }
    DoubleMinMax mm = new DoubleMinMax();
    for(DBID id : relation.iterDBIDs()) {
      double val = scoresult.get(id); // scores.get(id);
      val = scaling.getScaled(val);
      scores.putDouble(id, val);
      mm.put(val);
    }
    meta = new BasicOutlierScoreMeta(mm.getMin(), mm.getMax());
    or = new OutlierResult(meta, scoresult);

View Full Code Here


      logger.verbose("Computing the kNN outlier degree (distance to the k nearest neighbor)");
    }
    FiniteProgress progressKNNDistance = logger.isVerbose() ? new FiniteProgress("kNN distance for objects", relation.size(), logger) : null;

    DoubleMinMax minmax = new DoubleMinMax();
    WritableDoubleDataStore knno_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    // compute distance to the k nearest neighbor.
    for(DBID id : relation.iterDBIDs()) {
      // distance to the kth nearest neighbor
      final KNNResult<D> knns = knnQuery.getKNNForDBID(id, k);
      double dkn = knns.getKNNDistance().doubleValue();
      knno_score.putDouble(id, dkn);

      minmax.put(dkn);

      if(progressKNNDistance != null) {
        progressKNNDistance.incrementProcessed(logger);
View Full Code Here

    KNNQuery<O, D> knnQuery = database.getKNNQuery(distFunc, k);

    // track the maximum value for normalization
    DoubleMinMax ldofminmax = new DoubleMinMax();
    // compute the ldof values
    WritableDoubleDataStore ldofs = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);

    // compute LOF_SCORE of each db object
    if(logger.isVerbose()) {
      logger.verbose("Computing LDOFs");
    }
    FiniteProgress progressLDOFs = logger.isVerbose() ? new FiniteProgress("LDOF_SCORE for objects", relation.size(), logger) : null;

    for(DBID id : relation.iterDBIDs()) {
      KNNResult<D> neighbors = knnQuery.getKNNForDBID(id, k);
      int nsize = neighbors.size() - 1;
      // skip the point itself
      double dxp = 0;
      double Dxp = 0;
      for(DistanceResultPair<D> neighbor1 : neighbors) {
        if(!neighbor1.getDBID().equals(id)) {
          dxp += neighbor1.getDistance().doubleValue();
          for(DistanceResultPair<D> neighbor2 : neighbors) {
            if(!neighbor1.getDBID().equals(neighbor2.getDBID()) && !neighbor2.getDBID().equals(id)) {
              Dxp += distFunc.distance(neighbor1.getDBID(), neighbor2.getDBID()).doubleValue();
            }
          }
        }
      }
      dxp /= nsize;
      Dxp /= (nsize * (nsize - 1));
      Double ldof = dxp / Dxp;
      if(ldof.isNaN() || ldof.isInfinite()) {
        ldof = 1.0;
      }
      ldofs.putDouble(id, ldof);
      // update maximum
      ldofminmax.put(ldof);

      if(progressLDOFs != null) {
        progressLDOFs.incrementProcessed(logger);
View Full Code Here

    // Compute LRDs
    if(stepprog != null) {
      stepprog.beginStep(2, "Computing LRDs.", logger);
    }
    WritableDoubleDataStore lrds = computeLRDs(ids, kNNReach);

    // compute LOF_SCORE of each db object
    if(stepprog != null) {
      stepprog.beginStep(3, "Computing LOFs.", logger);
    }
    Pair<WritableDoubleDataStore, DoubleMinMax> lofsAndMax = computeLOFs(ids, lrds, kNNRefer);
    WritableDoubleDataStore lofs = lofsAndMax.getFirst();
    // track the maximum value for normalization.
    DoubleMinMax lofminmax = lofsAndMax.getSecond();

    if(stepprog != null) {
      stepprog.setCompleted(logger);
View Full Code Here

   * @param knnReach the precomputed neighborhood of the objects w.r.t. the
   *        reachability distance
   * @return the LRDs of the objects
   */
  protected WritableDoubleDataStore computeLRDs(DBIDs ids, KNNQuery<O, D> knnReach) {
    WritableDoubleDataStore lrds = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
    FiniteProgress lrdsProgress = logger.isVerbose() ? new FiniteProgress("LRD", ids.size(), logger) : null;
    for(DBID id : ids) {
      double sum = 0;
      KNNResult<D> neighbors = knnReach.getKNNForDBID(id, k);
      int nsize = neighbors.size() - (objectIsInKNN ? 0 : 1);
      for(DistanceResultPair<D> neighbor : neighbors) {
        if(objectIsInKNN || !neighbor.getDBID().equals(id)) {
          KNNResult<D> neighborsNeighbors = knnReach.getKNNForDBID(neighbor.getDBID(), k);
          sum += Math.max(neighbor.getDistance().doubleValue(), neighborsNeighbors.getKNNDistance().doubleValue());
        }
      }
      // Avoid division by 0
      double lrd = (sum > 0) ? nsize / sum : 0.0;
      lrds.putDouble(id, lrd);
      if(lrdsProgress != null) {
        lrdsProgress.incrementProcessed(logger);
      }
    }
    if(lrdsProgress != null) {
View Full Code Here

   * @param knnRefer the precomputed neighborhood of the objects w.r.t. the
   *        reference distance
   * @return the LOFs of the objects and the maximum LOF
   */
  protected Pair<WritableDoubleDataStore, DoubleMinMax> computeLOFs(DBIDs ids, DataStore<Double> lrds, KNNQuery<O, D> knnRefer) {
    WritableDoubleDataStore lofs = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_STATIC);
    // track the maximum value for normalization.
    DoubleMinMax lofminmax = new DoubleMinMax();

    FiniteProgress progressLOFs = logger.isVerbose() ? new FiniteProgress("LOF_SCORE for objects", ids.size(), logger) : null;
    for(DBID id : ids) {
      double lrdp = lrds.get(id);
      final double lof;
      if(lrdp > 0) {
        final KNNResult<D> neighbors = knnRefer.getKNNForDBID(id, k);
        int nsize = neighbors.size() - (objectIsInKNN ? 0 : 1);
        // skip the point itself
        // neighbors.remove(0);
        double sum = 0;
        for(DistanceResultPair<D> neighbor : neighbors) {
          if(objectIsInKNN || !neighbor.getDBID().equals(id)) {
            sum += lrds.get(neighbor.getDBID());
          }
        }
        lof = (sum / nsize) / lrdp;
      }
      else {
        lof = 1.0;
      }
      lofs.putDouble(id, lof);
      // update minimum and maximum
      lofminmax.put(lof);

      if(progressLOFs != null) {
        progressLOFs.incrementProcessed(logger);
View Full Code Here

    // Finalize covariance matrix:
    Vector mean = covmaker.getMeanVector();
    Matrix cmati = covmaker.destroyToSampleMatrix().inverse();

    DoubleMinMax minmax = new DoubleMinMax();
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(attributes.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(DBID id : attributes.iterDBIDs()) {
      Vector temp = deltas.get(id).minus(mean);
      final double score = temp.transposeTimesTimes(cmati, temp);
      minmax.put(score);
      scores.putDouble(id, score);
    }

    Relation<Double> scoreResult = new MaterializedRelation<Double>("mean multiple attributes spatial outlier", "mean-multipleattributes-outlier", TypeUtil.DOUBLE, scores, attributes.getDBIDs());
    OutlierScoreMeta scoreMeta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 0);
    OutlierResult or = new OutlierResult(scoreMeta, scoreResult);
View Full Code Here

      similarityVectors.put(id, sim);
    }
    E = null;
    // compute the relevance scores between specified Object and its neighbors
    DoubleMinMax minmax = new DoubleMinMax();
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(spatial.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(int i = 0; i < ids.size(); i++) {
      DBID id = ids.get(i);
      double gmean = 1.0;
      int cnt = 0;
      for(DBID n : neighbors.get(id)) {
        if(id.equals(n)) {
          continue;
        }
        double sim = MathUtil.angle(similarityVectors.get(id), similarityVectors.get(n));
        gmean *= sim;
        cnt++;
      }
      final double score = Math.pow(gmean, 1.0 / cnt);
      minmax.put(score);
      scores.putDouble(id, score);
    }

    Relation<Double> scoreResult = new MaterializedRelation<Double>("randomwalkec", "RandomWalkEC", TypeUtil.DOUBLE, scores, relation.getDBIDs());
    OutlierScoreMeta scoreMeta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 0.0);
    return new OutlierResult(scoreMeta, scoreResult);
View Full Code Here

    // KNNS
    WritableDataStore<ModifiableDBIDs> knns = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_TEMP | DataStoreFactory.HINT_HOT, ModifiableDBIDs.class);
    // RNNS
    WritableDataStore<ModifiableDBIDs> rnns = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_TEMP | DataStoreFactory.HINT_HOT, ModifiableDBIDs.class);
    // density
    WritableDoubleDataStore density = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_TEMP | DataStoreFactory.HINT_HOT);
    // init knns and rnns
    for(DBID id : relation.iterDBIDs()) {
      knns.put(id, DBIDUtil.newArray());
      rnns.put(id, DBIDUtil.newArray());
    }

    // TODO: use kNN preprocessor?
    KNNQuery<O, D> knnQuery = database.getKNNQuery(distFunc, k, DatabaseQuery.HINT_HEAVY_USE);

    for(DBID id : relation.iterDBIDs()) {
      // if not visited count=0
      int count = rnns.get(id).size();
      ModifiableDBIDs s;
      if(!processedIDs.contains(id)) {
        // TODO: use exactly k neighbors?
        KNNResult<D> list = knnQuery.getKNNForDBID(id, k);
        knns.get(id).addDBIDs(list.asDBIDs());
        processedIDs.add(id);
        s = knns.get(id);
        density.putDouble(id, 1 / list.get(k - 1).getDistance().doubleValue());

      }
      else {
        s = knns.get(id);
      }
      for(DBID q : s) {
        if(!processedIDs.contains(q)) {
          // TODO: use exactly k neighbors?
          KNNResult<D> listQ = knnQuery.getKNNForDBID(q, k);
          knns.get(q).addDBIDs(listQ.asDBIDs());
          density.putDouble(q, 1 / listQ.getKNNDistance().doubleValue());
          processedIDs.add(q);
        }

        if(knns.get(q).contains(id)) {
          rnns.get(q).add(id);
          rnns.get(id).add(q);
          count++;
        }
      }
      if(count >= s.size() * m) {
        pruned.add(id);
      }
    }

    // Calculate INFLO for any Object
    // IF Object is pruned INFLO=1.0
    DoubleMinMax inflominmax = new DoubleMinMax();
    WritableDoubleDataStore inflos = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(DBID id : relation.iterDBIDs()) {
      if(!pruned.contains(id)) {
        ModifiableDBIDs knn = knns.get(id);
        ModifiableDBIDs rnn = rnns.get(id);

        double denP = density.doubleValue(id);
        knn.addDBIDs(rnn);
        double den = 0;
        for(DBID q : knn) {
          double denQ = density.doubleValue(q);
          den = den + denQ;
        }
        den = den / rnn.size();
        den = den / denP;
        inflos.putDouble(id, den);
        // update minimum and maximum
        inflominmax.put(den);

      }
      if(pruned.contains(id)) {
        inflos.putDouble(id, 1.0);
        inflominmax.put(1.0);
      }
    }

    // Build result representation.
View Full Code Here

    DoubleMinMax minmax = new DoubleMinMax();

    // compute distance to the k nearest neighbor. n objects with the highest
    // distance are flagged as outliers
    WritableDoubleDataStore knnw_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    for(DBID id : relation.iterDBIDs()) {
      // compute sum of the distances to the k nearest neighbors

      final KNNResult<D> knn = knnQuery.getKNNForDBID(id, k);
      double skn = 0;
      for(DistanceResultPair<D> r : knn) {
        skn += r.getDistance().doubleValue();
      }
      knnw_score.putDouble(id, skn);
      minmax.put(skn);

      if(progressKNNWeight != null) {
        progressKNNWeight.incrementProcessed(logger);
      }
View Full Code Here

TOP

Related Classes of de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.