Package org.apache.jena.riot.tokens

Examples of org.apache.jena.riot.tokens.Tokenizer

324
325
326
327
328
329
330
331
        totalTuples += n ;
    }
   
    protected Tokenizer makeTokenizer(InputStream in)
    {
        Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ;
        return tokenizer ;
    }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
            System.exit(1) ;
        }
        for ( String filename : args )
        {
            InputStream in = IO.openFile(args[0]) ;
            Tokenizer tokenize = TokenizerFactory.makeTokenizerUTF8(in) ;
            Timer timer = new Timer() ;
            long count = 0 ;
            timer.startTimer() ;
            for ( ; tokenize.hasNext() ; )
            {
                Token t = tokenize.next() ;
                if ( print )
                    System.out.println(t) ;
                count++ ;
            }
            tokenize.close();
            long millis = timer.endTimer() ;
            if ( timing )
            {
                if ( millis == 0 )
                    System.out.printf("Tokens=%,d : Time=0.00s\n", count) ;
View Full Code Here
785
786
787
788
789
790
791
792
793
794
795
        if ( RDFLanguages.sameLang(RDFLanguages.RDFXML, lang) )
            parser = LangRDFXML.create(in, base, base, ErrorHandlerFactory.errorHandlerStd, output) ;
        else
        {
            Tokenizer tokenizer =
                RDFLanguages.RDFJSON.equals(lang)?
                              new TokenizerJSON(PeekReader.make(in)) :  
                              TokenizerFactory.makeTokenizer(in) ;
            parser = RiotReader.createParser(tokenizer, lang, base, output) ;
        }
View Full Code Here
797
798
799
800
801
802
803
804
805
    }
   
    // java.io.Readers are NOT preferred.
    private static void process(StreamRDF dest, String base, Reader in, Lang hintLang, Context context)
    {
        Tokenizer tokenizer = TokenizerFactory.makeTokenizer(in) ;
        LangRIOT parser = RiotReader.createParser(tokenizer, hintLang, base, dest) ;
        parser.parse() ;
    }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
    @Test public void bindingStream_63()              { testWriteRead(bb3) ; }

   
    static void testRead(String x, Binding ... bindings)
    {
        Tokenizer t = TokenizerFactory.makeTokenizerString(x) ;
        BindingInputStream inStream = new BindingInputStream(t) ;
       
        if ( bindings.length == 0 )
        {
            for ( ; inStream.hasNext() ; )
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
   
    @Override
    protected long parseCount(String... strings)
    {
        String string = StrUtils.strjoin("\n", strings) ;
        Tokenizer tokenizer = tokenizer(string) ;
        StreamRDFCounting sink = StreamRDFLib.count() ;
        LangNTriples x = RiotReader.createParserNTriples(tokenizer, sink) ;
        x.getProfile().setHandler(new ErrorHandlerEx()) ;
        x.parse() ;
        return sink.count() ;
View Full Code Here
493
494
495
496
497
498
499
500
501
  @Test(expected=IllegalArgumentException.class)
  public void rdfjson_invalid_tokenizer()
  {
    byte b[] = StrUtils.asUTF8bytes("") ;
    ByteArrayInputStream in = new ByteArrayInputStream(b);
    Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ;
        StreamRDFCounting sink = StreamRDFLib.count() ;
    LangRDFJSON parser = RiotReader.createParserRdfJson(tokenizer, sink) ;
  }
View Full Code Here
500
501
502
503
504
505
506
507
508
509
510
    LangRDFJSON parser = RiotReader.createParserRdfJson(tokenizer, sink) ;
  }

    private long parseCount(String string)
    {
        Tokenizer tokenizer = tokenizer(string) ;
        StreamRDFCounting sink = StreamRDFLib.count() ;
        LangRDFJSON x = RiotReader.createParserRdfJson(tokenizer, sink) ;
        x.getProfile().setHandler(new ErrorHandlerEx()) ;
        x.parse() ;
        return sink.count() ;
View Full Code Here
528
529
530
531
532
533
534
535
    private Tokenizer tokenizer(String str)
    {
        byte b[] = StrUtils.asUTF8bytes(str) ;
        ByteArrayInputStream in = new ByteArrayInputStream(b) ;
        Tokenizer tokenizer = new TokenizerJSON(PeekReader.makeUTF8(in)) ;
        return tokenizer ;
    }
View Full Code Here
161
162
163
164
165
166
167
168
    {
        //Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(string) ;
        // ASCII
        byte b[] = StrUtils.asUTF8bytes(string) ;
        ByteArrayInputStream in = new ByteArrayInputStream(b) ;
        Tokenizer tokenizer = TokenizerFactory.makeTokenizerASCII(in) ;
        return tokenizer ;
    }
View Full Code Here
TOP

Related Classes of org.apache.jena.riot.tokens.Tokenizer

  • com.hp.hpl.jena.sparql.engine.binding.TestBindingStreams
  • com.hp.hpl.jena.sparql.util.NodeFactory
  • com.hp.hpl.jena.sparql.util.NodeFactoryExtra
  • com.hp.hpl.jena.tdb.nodetable.NodecSSE
  • org.apache.jena.fuseki.validation.DataValidator
  • org.apache.jena.riot.lang.TestLangNTriples
  • org.apache.jena.riot.lang.TestLangNTuples
  • org.apache.jena.riot.lang.TestLangRdfJson
  • org.apache.jena.riot.lang.TestLangTrig
  • org.apache.jena.riot.lang.TestLangTurtle
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.