Package org.fnlp.ml.types.alphabet

Examples of org.fnlp.ml.types.alphabet.LabelAlphabet


    int ENG_all = 0, ENG_right = 0;
    Loss loss = new HammingLoss();

    String[][] labelsSet = new String[testSet.size()][];
    String[][] targetSet = new String[testSet.size()][];
    LabelAlphabet labels = cl.getAlphabetFactory().buildLabelAlphabet(
        "labels");
    for (int i = 0; i < testSet.size(); i++) {
      Instance carrier = testSet.get(i);
      int[] pred = (int[]) cl.classify(carrier).getLabel(0);
      if (acc) {
        len += pred.length;
        double e = loss.calc(carrier.getTarget(), pred);
        error += e;
        if(e != 0)
          senError++;
        //测试中英混杂语料
        if(hasENG) {
          String[][] origin = (String[][])carrier.getSource();
          int[] target = (int[])carrier.getTarget();
          for(int j = 0; j < target.length; j++) {
            if(origin[j][0].contains("ENG")) {
              ENG_all++;
              if(target[j] == pred[j])
                ENG_right++;
            }
          }
        }
      }
      labelsSet[i] = labels.lookupString(pred);
      targetSet[i] = labels.lookupString((int[])carrier.getTarget());
    }

    long endtime = System.currentTimeMillis();
    System.out.println("totaltime\t" + (endtime - starttime) / 1000.0);
    System.out.println("feature\t" + (featuretime - starttime) / 1000.0);
View Full Code Here


    int ENG_all = 0, ENG_right = 0;
    Loss loss = new HammingLoss();

    String[][] labelsSet = new String[testSet.size()][];
    String[][] targetSet = new String[testSet.size()][];
    LabelAlphabet labels = cl.getAlphabetFactory().buildLabelAlphabet(
        "labels");
    for (int i = 0; i < testSet.size(); i++) {
      Instance carrier = testSet.get(i);
      int[] pred = (int[]) cl.classify(carrier).getLabel(0);
      if (acc) {
        len += pred.length;
        double e = loss.calc(carrier.getTarget(), pred);
        error += e;
        if(e != 0)
          senError++;
        //测试中英混杂语料
        if(hasENG) {
          String[][] origin = (String[][])carrier.getSource();
          int[] target = (int[])carrier.getTarget();
          for(int j = 0; j < target.length; j++) {
            if(origin[j][0].contains("ENG")) {
              ENG_all++;
              if(target[j] == pred[j])
                ENG_right++;
            }
          }
        }
      }
      labelsSet[i] = labels.lookupString(pred);
      targetSet[i] = labels.lookupString((int[])carrier.getTarget());
    }

    long endtime = System.currentTimeMillis();
    System.out.println("totaltime\t" + (endtime - starttime) / 1000.0);
    System.out.println("feature\t" + (featuretime - starttime) / 1000.0);
View Full Code Here

    CoNLLReader reader = new CoNLLReader(file);

    BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
        new FileOutputStream(fp), charset));
   
    LabelAlphabet postagAlphabet = factory.buildLabelAlphabet("postag");

    int count = 0;
    while (reader.hasNext()) {

      Sentence instance = (Sentence) reader.next();
      int[] heads = (int[]) instance.getTarget();
      ParsingState state = new ParsingState(instance,factory);
      while (!state.isFinalState()) {
        // 左右焦点词在句子中的位置
        int[] lr = state.getFocusIndices();

        HashSparseVector features = state.getFeatures();
        ParsingState.Action action = getAction(lr[0], lr[1],
            heads);
        state.next(action);
        if (action == ParsingState.Action.LEFT)
          heads[lr[1]] = -1;
        if (action == ParsingState.Action.RIGHT)
          heads[lr[0]] = -1;

        // writer.write(String.valueOf(instance.postags[lr[0]]));
        String pos = instance.getTagAt(lr[0]);
        postagAlphabet.lookupIndex(pos);
        writer.write(pos);
        writer.write(" ");
        switch (action) {
        case LEFT:
          writer.write("L");
View Full Code Here

    fp = File.createTempFile("train-features", null, new File("./tmp/"));

    buildInstanceList(dataFile);

    LabelAlphabet postagAlphabet = factory.buildLabelAlphabet("postag");

    IFeatureAlphabet features = factory.DefaultFeatureAlphabet();

    SFGenerator generator = new SFGenerator();
    Linear[] models = new Linear[postagAlphabet.size()];
    int fsize = features.size();

    for (int i = 0; i < postagAlphabet.size(); i++) {
      String pos = postagAlphabet.lookupString(i);
      InstanceSet instset = readInstanceSet(pos);
      LabelAlphabet alphabet = factory.buildLabelAlphabet(pos);
      int ysize = alphabet.size();
      System.out.printf("Training with data: %s\n", pos);
      System.out.printf("Number of labels: %d\n", ysize);
      LinearMax solver = new LinearMax(generator, ysize);
      ZeroOneLoss loss = new ZeroOneLoss();
      Update update = new LinearMaxPAUpdate(loss);
View Full Code Here

   */
  private InstanceSet readInstanceSet(String pos) throws IOException {

    InstanceSet instset = new InstanceSet();

    LabelAlphabet labelAlphabet = factory.buildLabelAlphabet(pos);

    BufferedReader in = new BufferedReader(new InputStreamReader(
        new FileInputStream(fp), charset));

    String line = null;
    while ((line = in.readLine()) != null) {
      line = line.trim();
      if (line.matches("^$"))
        continue;
      if (line.startsWith(pos + " ")) {
        List<String> tokens = Arrays.asList(line.split("\\s+"));

        int[] data = new int[tokens.size() - 2];
        for (int i = 0; i < data.length; i++) {
          data[i] = Integer.parseInt(tokens.get(i + 2));
        }
        Instance inst = new Instance(data);
        inst.setTarget(labelAlphabet.lookupIndex(tokens.get(1)));

        instset.add(inst);
      }
    }

    in.close();

    labelAlphabet.setStopIncrement(true);
    instset.setAlphabetFactory(factory);

    return instset;
  }
View Full Code Here

    FNLPReader reader = new FNLPReader(file);
    FNLPReader preReader = new FNLPReader(file);
    InstanceSet instset = new InstanceSet();
   
    LabelAlphabet la = factory.DefaultLabelAlphabet();
    IFeatureAlphabet fa = factory.DefaultFeatureAlphabet();
    int count = 0;
   
    //preReader为了把ysize定下来
    la.lookupIndex("S");
    while(preReader.hasNext()){
      Sentence sent = (Sentence) preReader.next();
      Target targets = (Target)sent.getTarget();
      for(int i=0; i<sent.length(); i++){
        String label;
        if(targets.getHead(i) != -1){
          if(targets.getHead(i) < i){
            label = "L" + targets.getDepClass(i);
          }
          //else if(targets.getHead(i) > i){
          else{
            label = "R" + targets.getDepClass(i);
          }
          la.lookupIndex(label);
        }
      }
    }
    int ysize = la.size();
    la.setStopIncrement(true);
       
    while (reader.hasNext()) {
      Sentence sent = (Sentence) reader.next();
      //  int[] heads = (int[]) instance.getTarget();
      String depClass = null;
      Target targets = (Target)sent.getTarget();
      JointParsingState state = new JointParsingState(sent);
     
      while (!state.isFinalState()) {
        // 左右焦点词在句子中的位置
        int[] lr = state.getFocusIndices();

        ArrayList<String> features = state.getFeatures();
        JointParsingState.Action action = getAction(lr[0], lr[1],
            targets);
        switch (action) {
        case LEFT:
          depClass = targets.getDepClass(lr[1]);
          break;
        case RIGHT:
          depClass = targets.getDepClass(lr[0]);
          break;
        default:

        }
        state.next(action,depClass);
        if (action == JointParsingState.Action.LEFT)
          targets.setHeads(lr[1],-1);
        if (action == JointParsingState.Action.RIGHT)
          targets.setHeads(lr[0],-1);
        String label = "";
        switch (action) {
        case LEFT:
          label += "L"+sent.getDepClass(lr[1]);   
          break;
        case RIGHT:
          label+="R"+sent.getDepClass(lr[0]);
          break;
        default:
          label = "S";         
        }
        int id = la.lookupIndex(label);       
        Instance inst = new Instance();
        inst.setTarget(id);
        int[] idx = JointParser.addFeature(fa, features, ysize);
        inst.setData(idx);
        instset.add(inst);
View Full Code Here

    IFeatureAlphabet features = factory.DefaultFeatureAlphabet();

    SFGenerator generator = new SFGenerator();
    int fsize = features.size();
   
    LabelAlphabet la = factory.DefaultLabelAlphabet();
    int ysize = la.size();
    System.out.printf("开始训练");
    LinearMax solver = new LinearMax(generator, ysize);
    ZeroOneLoss loss = new ZeroOneLoss();
    Update update = new LinearMaxPAUpdate(loss);
    OnlineTrainer trainer = new OnlineTrainer(solver, update, loss,
View Full Code Here


    Loss loss = new HammingLoss();


    LabelAlphabet la = cl.getAlphabetFactory().DefaultLabelAlphabet();

    Predict xx = cl.classify(inst,5);
    double[] scores=new double[5];
    String[] sentenceout=new String[5];
    int[] pred;
    String[][] labelsSet = new String[xx.size()][];
    ArrayList<String> res = null;
    for(int j=0;j<xx.size();j++){
      pred = (int[]) xx.getLabel(j);
      float uuu = xx.getScore(j);
      labelsSet [j] = la.lookupString(pred);
      res = FormatCWS.toList(inst, labelsSet [j]);
//      System.out.println(res);
    }

   
View Full Code Here

import gnu.trove.map.hash.TIntObjectHashMap;

public class YamadaOptimization {

  private static void refineModels(Linear[] models, AlphabetFactory factory) {
    LabelAlphabet postags = factory.buildLabelAlphabet("postag");
    int posize = postags.size();
    float[][] nweights = new float[posize][];
    TFloatArrayList[] ww = new TFloatArrayList[posize];
    for (int i = 0; i < posize; i++) {
      nweights[i] = models[i].getWeights();
      ww[i] = new TFloatArrayList();
View Full Code Here


  public static HashSparseVector[] mean (InstanceSet trainingList,Tree tree)
  {

    LabelAlphabet alphabet = trainingList.getAlphabetFactory().DefaultLabelAlphabet();
    int numLabels = alphabet.size();
    HashSparseVector[] means = new  HashSparseVector[numLabels];
    int[] classNum = new int[numLabels];

    for(int i=0;i<numLabels;i++){
      means[i]=new HashSparseVector();
View Full Code Here

TOP

Related Classes of org.fnlp.ml.types.alphabet.LabelAlphabet

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.