Examples of TextTokenizer

Examples of com.fb2pdf.hadoop.TextTokenizer

51
52
53
54
55
56
57
58
59
60
61
62
63
64
        {
            if(excluded.contains(key.toString()))
                return;
           
            String line = value.toString().trim();
            TextTokenizer st = new TextTokenizer(line);
            while(st.hasMoreTokens())
            {
                String k = st.nextToken().trim().toLowerCase();
                if(k.length()>1)
                {
                    word.set(k);
                    output.collect(word, one);
                }
View Full Code Here

Examples of com.github.pmerienne.trident.ml.preprocessing.TextTokenizer

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
  @SuppressWarnings("unchecked")
  @Test
  public void test() {
    // Given
    TextTokenizer tokenizer = new EnglishTokenizer();
    List<String> d1 = tokenizer.tokenize(DATABASE_WIKI);
    List<String> d2 = tokenizer.tokenize(NOSQL_WIKI);
    List<String> d3 = tokenizer.tokenize(MYSQL_WIKI);
    List<String> d4 = tokenizer.tokenize(FLOWER_WIKI);
    List<String> d5 = tokenizer.tokenize(LILIUM_WIKI);
    List<String> d6 = tokenizer.tokenize(ROSE_WIKI);
    List<List<String>> training = Arrays.asList(d1, d2, d4, d5);

    TFIDF tfidf = new TFIDF();

    // When
View Full Code Here

Examples of org.auraframework.impl.util.TextTokenizer

93
94
95
96
97
98
99
100
101
102
103
104
                    // more information there, or we successfully finished, in which
                    // case it is rather unclear how this could happen.
                }
            }

            TextTokenizer tt = TextTokenizer.tokenize(contents, getLocation());
            tt.addExpressionRefs(this);

            return createDefinition(map);
        } catch (QuickFixException qfe) {
            return createDefinition(qfe);
        } catch (JsonParseException pe) {
View Full Code Here

Examples of org.auraframework.impl.util.TextTokenizer

73
74
75
76
77
78
79
80
81
82
83
    @Override
    protected AttributeDefRefImpl createDefinition() throws QuickFixException {
        if (AuraTextUtil.isNullEmptyOrWhitespace(stringValue)) {
            builder.setValue(children);
        } else {
            TextTokenizer tt = TextTokenizer.tokenize(stringValue, getLocation());
            builder.setValue(tt.asValue(getParentHandler()));
        }

        return builder.build();
    }
View Full Code Here

Examples of org.auraframework.impl.util.TextTokenizer

77
78
79
80
81
82
83
84
85
        builder.setDescription(getAttributeValue(ATTRIBUTE_DESCRIPTION));
    }

    @Override
    protected VarDef createDefinition() throws QuickFixException {
        TextTokenizer tt = TextTokenizer.tokenize(value, getLocation());
        builder.setValue(tt.asValue(getParentHandler()));
        return builder.build();
    }
View Full Code Here

Examples of org.auraframework.impl.util.TextTokenizer

143
144
145
146
147
148
149
150
151
152
153
154
        Object defaultObj = null;
        if (defaultValue != null) { // even it is an empty string or whitespace,
            // we should still set it in order to
            // distinguish from the case the default
            // value is not set at all.
            TextTokenizer tt = TextTokenizer.tokenize(defaultValue, getLocation());
            defaultObj = tt.asValue(getParentHandler());
        } else if (!body.isEmpty()) {
            defaultObj = body;
        }

        if (defaultObj != null) {
View Full Code Here

Examples of org.auraframework.impl.util.TextTokenizer

178
179
180
181
182
183
184
185
186
187
        String text = xmlReader.getText();
        boolean skip = getWhitespaceBehavior() == WhitespaceBehavior.OPTIMIZE ? AuraTextUtil
                .isNullEmptyOrWhitespace(text) : AuraTextUtil
                .isNullOrEmpty(text);
        if (!skip) {
            TextTokenizer tokenizer = TextTokenizer.tokenize(text,
                    getLocation(), getWhitespaceBehavior());
            body.addAll(tokenizer.asComponentDefRefs(this));
        }
    }
View Full Code Here

Examples of org.auraframework.impl.util.TextTokenizer

58
59
60
61
62
63
64
65
66
67
        boolean skip = getWhitespaceBehavior() == WhitespaceBehavior.OPTIMIZE ? AuraTextUtil
                .isNullEmptyOrWhitespace(text) : AuraTextUtil.isNullOrEmpty(text);

        if (!skip) {
            TextTokenizer tokenizer = TextTokenizer.tokenize(text, getLocation(), getWhitespaceBehavior());
            return tokenizer.asComponentDefRefs(parentHandler);
        }
        return Collections.emptyList();
    }
View Full Code Here

Examples of org.auraframework.impl.util.TextTokenizer

120
121
122
123
124
125
126
127
128
129
130
131
                String attValue = xmlReader.getAttributeValue(i);
                if (attributes.containsKey(att)) {
                    error("Duplicate values for attribute %s on tag %s", att, getTagName());
                }
                TextTokenizer tt = TextTokenizer.tokenize(attValue, getLocation());
                Object value = tt.asValue(getParentHandler());

                AttributeDefRefImpl.Builder atBuilder = new AttributeDefRefImpl.Builder();
                atBuilder.setDescriptor(att);
                atBuilder.setLocation(getLocation());
                atBuilder.setValue(value);
View Full Code Here

Examples of org.auraframework.impl.util.TextTokenizer

78
79
80
81
82
83
84
85
86
87
88
89
        builder.setDescriptor(SubDefDescriptorImpl.getInstance(container, parentDesc, LayoutItemDef.class));
        builder.setContainer(container);

        String action = getAttributeValue(ATTRIBUTE_ACTION);
        if (!AuraTextUtil.isNullEmptyOrWhitespace(action)) {
            TextTokenizer tt = TextTokenizer.tokenize(action, getLocation());
            builder.setAction(tt.asValue(getParentHandler()));
        }

        builder.setCache(getAttributeValue(ATTRIBUTE_CACHE));

        builder.setDescription(getAttributeValue(RootTagHandler.ATTRIBUTE_DESCRIPTION));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.