Package org.openjena.riot.tokens

Examples of org.openjena.riot.tokens.Tokenizer

364
365
366
367
368
369
370
371
        totalTuples += n ;
    }
   
    protected Tokenizer makeTokenizer(InputStream in)
    {
        Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ;
        return tokenizer ;
    }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
            System.exit(1) ;
        }
        for ( String filename : args )
        {
            InputStream in = IO.openFile(args[0]) ;
            Tokenizer tokenize = TokenizerFactory.makeTokenizerUTF8(in) ;
            Timer timer = new Timer() ;
            long count = 0 ;
            timer.startTimer() ;
            for ( ; tokenize.hasNext() ; )
            {
                Token t = tokenize.next() ;
                if ( print )
                    System.out.println(t) ;
                count++ ;
            }
            tokenize.close();
            long millis = timer.endTimer() ;
            if ( timing )
            {
                if ( millis == 0 )
                    System.out.printf("Tokens=%,d : Time=0.00s\n", count) ;
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
   
    @Override
    protected long parseCount(String... strings)
    {
        String string = StrUtils.strjoin("\n", strings) ;
        Tokenizer tokenizer = tokenizer(string) ;
        SinkCounting<Triple> sink = new SinkCounting<Triple>() ;
        LangNTriples x = RiotReader.createParserNTriples(tokenizer, sink) ;
        x.getProfile().setHandler(new ErrorHandlerEx()) ;
        x.parse() ;
        return sink.getCount() ;
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
    @Override
    protected void parseCheck(String... strings)
    {
        String string = StrUtils.strjoin("\n", strings) ;
        Tokenizer tokenizer = tokenizer(string) ;
        SinkCounting<Triple> sink = new SinkCounting<Triple>() ;
        LangNTriples x = RiotReader.createParserNTriples(tokenizer, sink) ;
        x.setProfile(RiotLib.profile(null, false, true, new ErrorHandlerEx())) ;
        x.parse() ;
    }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
    private static Graph parse(String ...strings)
    {
        String string = StrUtils.strjoin("\n", strings) ;
        Reader reader = new StringReader(string) ;
        String baseIRI = "http://base/" ;
        Tokenizer tokenizer = TokenizerFactory.makeTokenizer(reader) ;
       
        Graph graph = GraphFactory.createDefaultGraph() ;
        Sink<Triple> sink = RiotLoader.graphSink(graph) ;
       
        LangTurtle parser = RiotReader.createParserTurtle(tokenizer, "http://base/", sink) ;
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
   
  public static void parse(String testString)
  {
      // Need to access the prefix mapping.
     
      Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(testString) ;
      Sink<Triple> sink = new SinkNull<Triple>() ;

      LangTurtle parser = RiotReader.createParserTurtle(tokenizer, "http://base/", sink) ;
      PrefixMap prefixMap = parser.getProfile().getPrologue().getPrefixMap() ;

      prefixMap.add("a", "http://host/a#") ;
        prefixMap.add("x", "http://host/a#") ;
        // Unicode 00E9 is e-acute
        // Unicode 03B1 is alpha
        prefixMap.add("\u00E9", "http://host/e-acute/") ;
        prefixMap.add("\u03B1", "http://host/alpha/") ;
        prefixMap.add("", "http://host/") ;
        prefixMap.add("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#") ;
        prefixMap.add("xsd", "http://www.w3.org/2001/XMLSchema#") ;
        parser.parse();

        tokenizer.close();
  }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
116
117
            str = StrUtils.unescapeString(str) ;
            str = StrUtils.decodeHex(str, MarkerChar) ;
            return Node.createURI(str) ;
        }

        Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(str) ;
        if ( ! tokenizer.hasNext() )
            throw new TDBException("Failed to tokenise: "+str) ;
        Token t = tokenizer.next() ;

        try {
            Node n = t.asNode() ;
            if ( n == null ) throw new TDBException("Not a node: "+str) ;
            return n ;
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
            {
                if ( datafiles.size() > 0 )
                    cmdLog.info("Load: "+filename+" -- "+Utils.nowAsString()) ;
               
                InputStream in = IO.openFile(filename) ;
                Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ;
                ParserProfile profile = createParserProfile(runId, filename);
                Lang lang = Lang.guess(filename, Lang.NQUADS) ;
                if ( lang.isTriples() ) {
                    LangNTriples parser = new LangNTriples(tokenizer, profile, sink2) ;
                    parser.parse() ;
View Full Code Here
398
399
400
401
402
403
404
405
406
407
408
409
410
411
    }

    private static ParserProfile profile = createParserProfile();

    public static Node parse(String string) {
        Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(string) ;
        if ( ! tokenizer.hasNext() )
            return null ;
        Token t = tokenizer.next();
        Node n = profile.create(null, t) ;
        if ( tokenizer.hasNext() )
            Log.warn(RiotLib.class, "String has more than one token in it: "+string) ;
        return n ;
    }
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
173
174
    return offsets;
    }
   
    private static Node parse(String string) {
      ParserProfile profile = RiotLib.profile(Lang.NQUADS, null, null) ;
        Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(string) ;
        if ( ! tokenizer.hasNext() )
            return null ;
        Token t = tokenizer.next();
        Node n = profile.create(null, t) ;
        if ( tokenizer.hasNext() )
            Log.warn(RiotLib.class, "String has more than one token in it: "+string) ;
        return n ;
    }
View Full Code Here
TOP

Related Classes of org.openjena.riot.tokens.Tokenizer

  • com.hp.hpl.jena.sparql.engine.binding.TestBindingStreams
  • com.hp.hpl.jena.sparql.resultset.TSVInput
  • com.hp.hpl.jena.sparql.util.NodeFactory
  • com.hp.hpl.jena.tdb.nodetable.NodecSSE
  • org.apache.jena.fuseki.validation.DataValidator
  • org.apache.jena.tdbloader4.FirstReducer
  • org.apache.jena.tdbloader4.io.QuadRecordReader
  • org.apache.jena.tdbloader4.io.QuadWritable
  • org.apache.jena.tdbloader4.SecondReducer
  • org.openjena.riot.lang.TestLangNQuads
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.