Examples of Tokens

Examples of net.sf.okapi.steps.tokenization.tokens.Tokens

85
86
87
88
89
90
91
92
93
94
95
96
                        "can't understand '{}' as a BCP-47 locale; defaulting to English",
                        bcp47Locale);
                locale = LocaleId.ENGLISH;
            }

            Tokens tokens = StringTokenizer.tokenizeString(s, locale, "WORD");
            return tokens.size();
        } catch (Exception e) {
            Object[] args = new Object[] { s, bcp47Locale, e };
            log.error("unable to count words in string '{}' for locale '{}'",
                    args);
            return 0;
View Full Code Here

Examples of net.sourceforge.pmd.cpd.Tokens

35
36
37
38
39
40
41
42
43
44
45
  @Test
  public void shouldWorkOnValidInput() throws URISyntaxException {
    File file = new File(getClass().getResource("codechunks-project/code_chunks.cc").toURI());
    SourceCode source = new SourceCode(new SourceCode.FileCodeLoader(file, "key"));
    Tokens cpdTokens = new Tokens();
    CxxTokenizer tokenizer = new CxxTokenizer(Charset.forName("UTF-8"));
    tokenizer.tokenize(source, cpdTokens);
    List<TokenEntry> list = cpdTokens.getTokens();
    assertThat(list.size()).isEqualTo(371);
  }
View Full Code Here

Examples of net.sourceforge.pmd.cpd.Tokens

36
37
38
39
40
41
42
43
44
45
  @Test
  public void test() {
    JavaScriptTokenizer tokenizer = new JavaScriptTokenizer(Charset.forName("UTF-8"));
    SourceCode source = mock(SourceCode.class);
    when(source.getFileName()).thenReturn(new File("src/test/resources/cpd/Person.js").getAbsolutePath());
    Tokens tokens = new Tokens();
    tokenizer.tokenize(source, tokens);
    assertThat(tokens.getTokens().size()).isGreaterThan(1);
    assertThat(tokens.getTokens().get(tokens.size() - 1)).isEqualTo(TokenEntry.getEOF());
  }
View Full Code Here

Examples of net.sourceforge.pmd.cpd.Tokens

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    return blockBuilder.chunk(resourceId, chunk(file));
  }

  public List<TokensLine> chunk(File file) {
    SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(file, encoding));
    Tokens tokens = new Tokens();
    TokenEntry.clearImages();
    try {
      tokenizer.tokenize(sourceCode, tokens);
    } catch (IOException e) {
      throw Throwables.propagate(e);
    }
    TokenEntry.clearImages();
    return convert(tokens.getTokens());
  }
View Full Code Here

Examples of org.modeshape.common.text.TokenStream.Tokens

33
34
35
36
37
38
39
40
41
42
43
    @Before
    public void beforeEach() {
        tokenizer = new CndTokenizer(true, false);
        final LinkedList<int[]> tokenValues = new LinkedList<int[]>();
        tokenFactory = new Tokens() {
            @Override
            public void addToken( Position position,
                                  int index ) {
                int[] token = new int[] {index, index + 1, 0};
                tokenValues.add(token);
View Full Code Here

Examples of org.modeshape.common.text.TokenStream.Tokens

35
36
37
38
39
40
41
42
43
44
45
    @Before
    public void beforeEach() {
        tokenizer = TokenStream.basicTokenizer(true);
        final LinkedList<int[]> tokenValues = new LinkedList<int[]>();
        tokenFactory = new Tokens() {
            @Override
            public void addToken( Position position,
                                  int index ) {
                int[] token = new int[] {index, index + 1, 0};
                tokenValues.add(token);
View Full Code Here

Examples of org.modeshape.common.text.TokenStream.Tokens

34
35
36
37
38
39
40
41
42
43
44
    @Before
    public void beforeEach() {
        tokenizer = new SqlTokenizer(true);
        final LinkedList<int[]> tokenValues = new LinkedList<int[]>();
        tokenFactory = new Tokens() {
            @Override
            public void addToken( Position position,
                                  int index ) {
                int[] token = new int[] {index, index + 1, 0};
                tokenValues.add(token);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.