Examples of lex()


Examples of asia.redact.bracket.properties.PropertiesLexer.lex()

      if(locale.includes(info.locale)){
        FileReader in;
        try {
          in = new FileReader(file);
          PropertiesLexer lexer = new PropertiesLexer(in);
          lexer.lex();
          List<PropertiesToken> list = lexer.getList();
          new PropertiesParser(list, this).parse();
        } catch (FileNotFoundException e) {
          // fail cleanly and quickly as correct properties
          // are usually necessary to having non-deterministic behavior later
View Full Code Here

Examples of com.sonar.sslr.impl.Lexer.lex()

  }

  public final void tokenize(SourceCode source, Tokens cpdTokens) {
    Lexer lexer = CxxLexer.create(new CxxConfiguration(charset));
    String fileName = source.getFileName();
    List<Token> tokens = lexer.lex(new File(fileName));
    for (Token token : tokens) {
      TokenEntry cpdToken = new TokenEntry(getTokenImage(token), fileName, token.getLine());
      cpdTokens.add(cpdToken);
    }
    cpdTokens.add(TokenEntry.getEOF());
View Full Code Here

Examples of com.sonar.sslr.impl.Lexer.lex()

  @Override
  public final void tokenize(SourceCode source, Tokens cpdTokens) {
    Lexer lexer = EcmaScriptLexer.create(new EcmaScriptConfiguration(charset));
    String fileName = source.getFileName();
    List<Token> tokens = lexer.lex(new File(fileName));
    for (Token token : tokens) {
      TokenEntry cpdToken = new TokenEntry(getTokenImage(token), fileName, token.getLine());
      cpdTokens.add(cpdToken);
    }
    cpdTokens.add(TokenEntry.getEOF());
View Full Code Here

Examples of com.sonar.sslr.impl.Lexer.lex()

    }

    public void tokenize(SourceCode source, Tokens cpdTokens) throws IOException {
        Lexer lexer = ObjectiveCLexer.create(new ObjectiveCConfiguration(charset));
        String fileName = source.getFileName();
        List<Token> tokens = lexer.lex(new File(fileName));
        for (Token token : tokens) {
            TokenEntry cpdToken = new TokenEntry(getTokenImage(token), fileName, token.getLine());
            cpdTokens.add(cpdToken);
        }
        cpdTokens.add(TokenEntry.getEOF());
View Full Code Here

Examples of edu.stanford.nlp.parser.lexparser.EnglishTreebankParserParams.lex()

      lexOptions.unknownPrefixSize = 1;
      lexOptions.unknownSuffixSize = 1;
    }
    Index<String> wordIndex = new HashIndex<String>();
    Index<String> tagIndex = new HashIndex<String>();
    Lexicon lex = tlpp.lex(op, wordIndex, tagIndex);
   
    int computeAfter = (int) (0.50 * tb.size());
    Counter<String> vocab = new ClassicCounter<String>();
    Counter<String> unkCounter = new ClassicCounter<String>();
    int treeId = 0;
View Full Code Here

Examples of edu.stanford.nlp.parser.lexparser.TreebankLangParserParams.lex()

      lexOptions.unknownPrefixSize = 1;
      lexOptions.unknownSuffixSize = 1;
    }
    Index<String> wordIndex = new HashIndex<String>();
    Index<String> tagIndex = new HashIndex<String>();
    Lexicon lex = tlpp.lex(op, wordIndex, tagIndex);
   
    int computeAfter = (int) (0.50 * tb.size());
    Counter<String> vocab = new ClassicCounter<String>();
    Counter<String> unkCounter = new ClassicCounter<String>();
    int treeId = 0;
View Full Code Here

Examples of fri.patterns.interpreter.parsergenerator.lexer.LexerImpl.lex()

      String parseFile = args[i];
      Reader parseInput = new UnicodeReader(new FileInputStream(parseFile));
      lexer.setInput(parseInput);
     
      System.err.println("======================== Parsing: "+parseFile+" ========================");
      boolean result = lexer.lex(new PrintXmlLexerSemantic());
      System.err.println("========================================================");
     
      System.err.println("Lexing took "+timer.getInterval()+" millis.");
      System.err.println("Result was: "+result);
    }
View Full Code Here

Examples of lexer.Lexer.lex()

    lexer.addDescender("Parentheses", new Descender("(", ")", new Type<Token>("Parentheses"), null));
    lexer.addDescender("Brackets", new Descender("[", "]", new Type<Token>("Brackets"), null));
    lexer.ignore("\n");
    String test = "10.0 100 (3.0 300\n) [51 5 6] ()";
    try {
      System.out.println(test + " -> " + lexer.lex(test));
    }
    catch (LexerException e) {
      System.out.println(test + " -> " + e.getMessage());
      e.printStackTrace();
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.