Package com.googlecode.gaal.preprocess.impl

Examples of com.googlecode.gaal.preprocess.impl.LowerCaseNormalizer

69
70
71
72
73
74
75
76
77
78
79
80
81
    public VectorVisualizer(String srcFileName, String dstFileName, int windowSize) throws FileNotFoundException {
        FileReader srcReader = new FileReader(srcFileName);
        FileReader dstReader = new FileReader(dstFileName);
        Tokenizer<String> srcTokenizer = new MultidocumentRegexTokenizer(srcReader, STRING_REGEX,
                new LowerCaseNormalizer());
        Tokenizer<String> dstTokenizer = new MultidocumentRegexTokenizer(dstReader, STRING_REGEX,
                new LowerCaseNormalizer());
        srcCorpus = new TreeMapCorpus(srcTokenizer, SEPARATORS);
        dstCorpus = new TreeMapCorpus(dstTokenizer, SEPARATORS);
        // IntervalSetBuilder intervalSetBuilder = new SupermaximalSetBuilder();
        srcSequence = srcCorpus.sequence();
        dstSequence = dstCorpus.sequence();
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
                Arrays.asList(new String[] { ".", ",", ";", "", "(", ")", "of" }));
        Set<String> dstSeparators = new HashSet<String>(Arrays.asList(new String[] { ".", ",", ";", "(", ")", "von" }));
        FileReader srcReader = new FileReader(srcFileName);
        FileReader dstReader = new FileReader(dstFileName);
        Tokenizer<String> srcTokenizer = new MultidocumentRegexTokenizer(srcReader, STRING_REGEX, new StopWordRemover(
                srcStopWords, new LowerCaseNormalizer()));
        Tokenizer<String> dstTokenizer = new MultidocumentRegexTokenizer(dstReader, STRING_REGEX, new StopWordRemover(
                dstStopWords, new LowerCaseNormalizer()));
        Corpus<String> srcCorpus = new TreeMapCorpus(srcTokenizer, srcSeparators);
        Corpus<String> dstCorpus = new TreeMapCorpus(dstTokenizer, dstSeparators);

        double minSimilarity = 0.3;
        int alignmentsNumber = 3;
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
            } catch (FileNotFoundException e) {
                System.err.printf("can't open source file: ", e.getMessage());
                System.exit(1);
            }
            Tokenizer<String> srcTokenizer = new MultidocumentRegexTokenizer(srcReader, regex, new StopWordRemover(
                    srcStopWords, new LowerCaseNormalizer()));
            Corpus<String> srcCorpus = new TreeMapCorpus(srcTokenizer, srcSeparators, corpusSize);
            Corpus<String> dstCorpus = null;
            if (dstFileName != null) {
                try {
                    dstReader = new FileReader(dstFileName);
                } catch (FileNotFoundException e) {
                    System.err.printf("can't open target file: ", e.getMessage());
                    System.exit(1);
                }
                Tokenizer<String> dstTokenizer = new MultidocumentRegexTokenizer(dstReader, regex, new StopWordRemover(
                        dstStopWords, new LowerCaseNormalizer()));
                dstCorpus = new TreeMapCorpus(dstTokenizer, dstSeparators, corpusSize);
            }
            ANALYSER = new ConcurrentAnalyser(srcCorpus, dstCorpus);
        }
        return ANALYSER;
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
        } catch (FileNotFoundException e) {
            System.err.printf("can't open source file: ", e.getMessage());
            System.exit(1);
        }
        Tokenizer<String> tokenizer = new MultidocumentRegexTokenizer(srcReader, regex, new StopWordRemover(stopWords,
                new LowerCaseNormalizer()));
        Iterator<Document<String>> docIter = tokenizer.iterator();
        int lineCounter = 0;
        while (docIter.hasNext()) {
            Document<String> doc = docIter.next();
            System.out.format("\nDocument #%d:\n", doc.getId());
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
        // String[] alphabet = { null, "a", "ā", "b", "g", "h", "j", "l", "m",
        // "n", "r", "s", "t", "y", "#" };
        // StringReader sr = new StringReader("caggtcagtcacggtatca#");
        // String[] alphabet = { null, "a", "c", "g", "t", "#" };

        Tokenizer<String> tokenizer = new RegexTokenizer(sr, "[\\W\\w]", new LowerCaseNormalizer());
        Corpus<String> corpus = new TreeMapCorpus(tokenizer, alphabet);
        System.out.println("alphabet size " + corpus.alphabetSize());
        System.out.println("text size " + corpus.sequence().size());
        System.out.println("text " + corpus.sequence());
        System.out.println("alphabet " + corpus.alphabet());
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    }

    @Test
    public void testTokenizer() {
        StringReader sr = new StringReader(TINY_CORPUS);
        Tokenizer<String> tokenizer = new RegexTokenizer(sr, CHAR_REGEX, new LowerCaseNormalizer());
        for (Document<String> doc : tokenizer) {
            int index = 0;
            for (String token : doc) {
                String expected = TINY_CORPUS.substring(index, index + 1);
                if (!expected.equals(token)) {
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
    public void testAlphabetSize() throws FileNotFoundException {
        assertTrue(corpus.alphabetSize() == TINY_CORPUS_ALPHABET.length - 1);

        FileReader reader = new FileReader("data/tlg.txt");

        Tokenizer<String> tokenizer = new RegexTokenizer(reader, Analyser.STRING_REGEX, new LowerCaseNormalizer());
        Corpus<String> corpus1 = new TreeMapCorpus(tokenizer);
        reader = new FileReader("data/tlg.txt");

        tokenizer = new RegexTokenizer(reader, Analyser.STRING_REGEX, new LowerCaseNormalizer());
        Corpus<String> corpus2 = new TreeMapCorpus(tokenizer, new HashSet<String>());
        System.out.println(corpus1.alphabetSize() + " " + corpus2.alphabetSize());
        // assertTrue(corpus1.alphabetSize() == ALPHABET_SIZE);
        assertTrue(corpus1.alphabetSize() == corpus2.alphabetSize());
    }
View Full Code Here
122
123
124
125
126
127
128
129
    public static Corpus<String> createMississippiCorpus() {
        String text = "mississippi#";
        String[] alphabet = { null, "i", "m", "p", "s", "#" };
        StringReader sr = new StringReader(text);

        Tokenizer<String> tokenizer = new RegexTokenizer(sr, CHAR_REGEX, new LowerCaseNormalizer());
        return new TreeMapCorpus(tokenizer, alphabet);
    }
View Full Code Here
131
132
133
134
135
136
137
138
    public static Corpus<String> createMiningEngineeringCorpus() {
        String text = "mining␣engineering#";
        String[] alphabet = { null, "e", "g", "i", "m", "n", "r", "␣", "#" };
        StringReader sr = new StringReader(text);

        Tokenizer<String> tokenizer = new RegexTokenizer(sr, CHAR_REGEX, new LowerCaseNormalizer());
        return new TreeMapCorpus(tokenizer, alphabet);
    }
View Full Code Here
138
139
140
141
142
143
144
145
    }

    public static Corpus<String> createTinyCorpus() {
        StringReader sr = new StringReader(TINY_CORPUS);

        Tokenizer<String> tokenizer = new RegexTokenizer(sr, CHAR_REGEX, new LowerCaseNormalizer());
        return new TreeMapCorpus(tokenizer, TINY_CORPUS_ALPHABET);
    }
View Full Code Here
TOP

Related Classes of com.googlecode.gaal.preprocess.impl.LowerCaseNormalizer

  • com.googlecode.gaal.analysis.impl.Analyser
  • com.googlecode.gaal.cli.Main
  • com.googlecode.gaal.cli.Test
  • com.googlecode.gaal.data.impl.CorpusTest
  • com.googlecode.gaal.vis.VectorVisualizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.