Package org.apache.mahout.math

Examples of org.apache.mahout.math.Matrix


   * @return The computed cumulative transition matrix.
   */
  public static Matrix getCumulativeTransitionMatrix(HmmModel model) {
    // fetch the needed parameters from the model
    int hiddenStates = model.getNrOfHiddenStates();
    Matrix transitionMatrix = model.getTransitionMatrix();
    // now compute the cumulative transition matrix
    Matrix resultMatrix = new DenseMatrix(hiddenStates, hiddenStates);
    for (int i = 0; i < hiddenStates; ++i) {
      double sum = 0;
      for (int j = 0; j < hiddenStates; ++j) {
        sum += transitionMatrix.get(i, j);
        resultMatrix.set(i, j, sum);
      }
      resultMatrix.set(i, hiddenStates - 1, 1.0);
      // make sure the last
      // state has always a
      // cumulative
      // probability of
      // exactly 1.0
View Full Code Here


   */
  public static Matrix getCumulativeOutputMatrix(HmmModel model) {
    // fetch the needed parameters from the model
    int hiddenStates = model.getNrOfHiddenStates();
    int outputStates = model.getNrOfOutputStates();
    Matrix outputMatrix = model.getEmissionMatrix();
    // now compute the cumulative output matrix
    Matrix resultMatrix = new DenseMatrix(hiddenStates, outputStates);
    for (int i = 0; i < hiddenStates; ++i) {
      double sum = 0;
      for (int j = 0; j < outputStates; ++j) {
        sum += outputMatrix.get(i, j);
        resultMatrix.set(i, j, sum);
      }
      resultMatrix.set(i, outputStates - 1, 1.0);
      // make sure the last
      // output state has
      // always a cumulative
      // probability of 1.0
    }
View Full Code Here

   *
   * @param model model to normalize
   */
  public static void normalizeModel(HmmModel model) {
    Vector ip = model.getInitialProbabilities();
    Matrix emission = model.getEmissionMatrix();
    Matrix transition = model.getTransitionMatrix();
    // check normalization for all probabilities
    double isum = 0;
    for (int i = 0; i < model.getNrOfHiddenStates(); ++i) {
      isum += ip.getQuick(i);
      double sum = 0;
      for (int j = 0; j < model.getNrOfHiddenStates(); ++j) {
        sum += transition.getQuick(i, j);
      }
      if (sum != 1.0) {
        for (int j = 0; j < model.getNrOfHiddenStates(); ++j) {
          transition.setQuick(i, j, transition.getQuick(i, j) / sum);
        }
      }
      sum = 0;
      for (int j = 0; j < model.getNrOfOutputStates(); ++j) {
        sum += emission.getQuick(i, j);
View Full Code Here

   * @param threshold minimum value a model entry must have to be retained.
   * @return Truncated model
   */
  public static HmmModel truncateModel(HmmModel model, double threshold) {
    Vector ip = model.getInitialProbabilities();
    Matrix em = model.getEmissionMatrix();
    Matrix tr = model.getTransitionMatrix();
    // allocate the sparse data structures
    RandomAccessSparseVector sparseIp = new RandomAccessSparseVector(model
        .getNrOfHiddenStates());
    SparseMatrix sparseEm = new SparseMatrix(new int[]{
        model.getNrOfHiddenStates(), model.getNrOfOutputStates()});
    SparseMatrix sparseTr = new SparseMatrix(new int[]{
        model.getNrOfHiddenStates(), model.getNrOfHiddenStates()});
    // now transfer the values
    for (int i = 0; i < model.getNrOfHiddenStates(); ++i) {
      double value = ip.getQuick(i);
      if (value > threshold) {
        sparseIp.setQuick(i, value);
      }
      for (int j = 0; j < model.getNrOfHiddenStates(); ++j) {
        value = tr.getQuick(i, j);
        if (value > threshold) {
          sparseTr.setQuick(i, j, value);
        }
      }

View Full Code Here

        "--symmetric", "true"
    };
    new DistributedLanczosSolver().new DistributedLanczosSolverJob().run(args);

    Path rawEigenvectors = new Path(output, DistributedLanczosSolver.RAW_EIGENVECTORS);
    Matrix eigenVectors = new DenseMatrix(10, corpus.numCols());
    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(rawEigenvectors.toUri(), conf);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, rawEigenvectors, conf);
    try {
      Writable key = reader.getKeyClass().asSubclass(Writable.class).newInstance();
      Writable value = reader.getValueClass().asSubclass(Writable.class).newInstance();
      int i = 0;
      while (reader.next(key, value)) {
        Vector v = ((VectorWritable) value).get();
        eigenVectors.assignRow(i, v);
        System.out.println("k=" + key.toString() + " V=" + AbstractCluster.formatVector(v, null));
        value = reader.getValueClass().asSubclass(Writable.class).newInstance();
        i++;
      }
      assertEquals("number of eigenvectors", 9, i);
View Full Code Here

        "--cleansvd", "true"
    };
    new DistributedLanczosSolver().new DistributedLanczosSolverJob().run(args);
 
    Path cleanEigenvectors = new Path(output, EigenVerificationJob.CLEAN_EIGENVECTORS);
    Matrix eigenVectors = new DenseMatrix(10, corpus.numCols());
    Configuration conf = new Configuration();
 
    FileSystem fs = FileSystem.get(cleanEigenvectors.toUri(), conf);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, cleanEigenvectors, conf);
    try {
      Writable key = reader.getKeyClass().asSubclass(Writable.class).newInstance();
      Writable value = reader.getValueClass().asSubclass(Writable.class).newInstance();
      int i = 0;
      while (reader.next(key, value)) {
        Vector v = ((VectorWritable) value).get();
        eigenVectors.assignRow(i, v);
        System.out.println("k=" + key.toString() + " V=" + AbstractCluster.formatVector(v, null));
        value = reader.getValueClass().asSubclass(Writable.class).newInstance();
        i++;
      }
      assertEquals("number of clean eigenvectors", 4, i);
View Full Code Here

    DistributedRowMatrix corpus = new TestDistributedRowMatrix().randomDistributedMatrix(500,
        450, 400, 10, 10.0, symmetric, testData.getAbsolutePath());
    corpus.configure(new JobConf());
    DistributedLanczosSolver solver = new DistributedLanczosSolver();
    int desiredRank = 30;
    Matrix eigenVectors = new DenseMatrix(desiredRank, corpus.numCols());
    List<Double> eigenValues = new ArrayList<Double>();
    solver.solve(corpus, desiredRank, eigenVectors, eigenValues, symmetric);
    assertOrthonormal(eigenVectors);
    assertEigen(eigenVectors, corpus, eigenVectors.numRows() / 2, 0.01, symmetric);
  }
View Full Code Here

      // compute the Viterbi path
      HmmAlgorithms.viterbiAlgorithm(viterbiPath, delta, phi, lastIteration,
          observedSequence, scaled);
      // Viterbi iteration uses the viterbi path to update
      // the probabilities
      Matrix emissionMatrix = iteration.getEmissionMatrix();
      Matrix transitionMatrix = iteration.getTransitionMatrix();

      // first, assign the pseudo count
      emissionMatrix.assign(pseudoCount);
      transitionMatrix.assign(pseudoCount);

      // now count the transitions
      countTransitions(transitionMatrix, emissionMatrix, observedSequence,
          viterbiPath);

      // and normalize the probabilities
      for (int j = 0; j < iteration.getNrOfHiddenStates(); ++j) {
        double sum = 0;
        // normalize the rows of the transition matrix
        for (int k = 0; k < iteration.getNrOfHiddenStates(); ++k) {
          sum += transitionMatrix.getQuick(j, k);
        }
        for (int k = 0; k < iteration.getNrOfHiddenStates(); ++k) {
          transitionMatrix
              .setQuick(j, k, transitionMatrix.getQuick(j, k) / sum);
        }
        // normalize the rows of the emission matrix
        sum = 0;
        for (int k = 0; k < iteration.getNrOfOutputStates(); ++k) {
          sum += emissionMatrix.getQuick(j, k);
View Full Code Here

      throw new UnknownError("Cloning HmmModels broke. Check for programming errors, changed APIs etc.");
    }
    // allocate space for baum-welch factors
    int hiddenCount = initialModel.getNrOfHiddenStates();
    int visibleCount = observedSequence.length;
    Matrix alpha = new DenseMatrix(visibleCount, hiddenCount);
    Matrix beta = new DenseMatrix(visibleCount, hiddenCount);

    // now run the baum Welch training iteration
    for (int it = 0; it < maxIterations; ++it) {
      // fetch emission and transition matrix of current iteration
      Vector initialProbabilities = iteration.getInitialProbabilities();
      Matrix emissionMatrix = iteration.getEmissionMatrix();
      Matrix transitionMatrix = iteration.getTransitionMatrix();

      // compute forward and backward factors
      HmmAlgorithms.forwardAlgorithm(alpha, iteration, observedSequence, scaled);
      HmmAlgorithms.backwardAlgorithm(beta, iteration, observedSequence, scaled);

      if (scaled) {
        logScaledBaumWelch(observedSequence, iteration, alpha, beta);
      } else {
        unscaledBaumWelch(observedSequence, iteration, alpha, beta);
      }
      // normalize transition/emission probabilities
      // and normalize the probabilities
      double isum = 0;
      for (int j = 0; j < iteration.getNrOfHiddenStates(); ++j) {
        double sum = 0;
        // normalize the rows of the transition matrix
        for (int k = 0; k < iteration.getNrOfHiddenStates(); ++k) {
          sum += transitionMatrix.getQuick(j, k);
        }
        for (int k = 0; k < iteration.getNrOfHiddenStates(); ++k) {
          transitionMatrix
              .setQuick(j, k, transitionMatrix.getQuick(j, k) / sum);
        }
        // normalize the rows of the emission matrix
        sum = 0;
        for (int k = 0; k < iteration.getNrOfOutputStates(); ++k) {
          sum += emissionMatrix.getQuick(j, k);
View Full Code Here

    return iteration;
  }

  private static void unscaledBaumWelch(int[] observedSequence, HmmModel iteration, Matrix alpha, Matrix beta) {
    Vector initialProbabilities = iteration.getInitialProbabilities();
    Matrix emissionMatrix = iteration.getEmissionMatrix();
    Matrix transitionMatrix = iteration.getTransitionMatrix();
    double modelLikelihood = HmmEvaluator.modelLikelihood(alpha, false);

    for (int i = 0; i < iteration.getNrOfHiddenStates(); ++i) {
      initialProbabilities.setQuick(i, alpha.getQuick(0, i)
          * beta.getQuick(0, i));
    }

    // recompute transition probabilities
    for (int i = 0; i < iteration.getNrOfHiddenStates(); ++i) {
      for (int j = 0; j < iteration.getNrOfHiddenStates(); ++j) {
        double temp = 0;
        for (int t = 0; t < observedSequence.length - 1; ++t) {
          temp += alpha.getQuick(t, i)
              * emissionMatrix.getQuick(j, observedSequence[t + 1])
              * beta.getQuick(t + 1, j);
        }
        transitionMatrix.setQuick(i, j, transitionMatrix.getQuick(i, j)
            * temp / modelLikelihood);
      }
    }
    // recompute emission probabilities
    for (int i = 0; i < iteration.getNrOfHiddenStates(); ++i) {
View Full Code Here

TOP

Related Classes of org.apache.mahout.math.Matrix

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.