Examples of detectCharset()

Examples of net.yacy.document.parser.html.ScraperInputStream.detectCharset()

955
956
957
958
959
960
961
962
963
964
965
                            if((path.endsWith("html") || path.endsWith("htm"))) {
                                // save position
                                fis.mark(1000);
                                // scrape document to look up charset
                                final ScraperInputStream htmlFilter = new ScraperInputStream(fis,"UTF-8",new DigestURI("http://localhost"),null,false);
                                final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
                                if(charset != null)
                                    mimeType = mimeType + "; charset="+charset;
                                // reset position
                                fis.reset();
                            }
View Full Code Here

Examples of net.yacy.document.parser.html.ScraperInputStream.detectCharset()

99
100
101
102
103
104
105
106
107
108
109
        // nothing found: try to find a meta-tag
        if (charset == null) {
            try {
                final ScraperInputStream htmlFilter = new ScraperInputStream(sourceStream,documentCharset,location,null,false);
                sourceStream = htmlFilter;
                charset = htmlFilter.detectCharset();
            } catch (IOException e1) {
                throw new Parser.Failure("Charset error:" + e1.getMessage(), location);
            }
        }
View Full Code Here

Examples of net.yacy.document.parser.html.ScraperInputStream.detectCharset()

1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
                            if((path.endsWith("html") || path.endsWith("htm"))) {
                                // save position
                                fis.mark(1000);
                                // scrape document to look up charset
                                final ScraperInputStream htmlFilter = new ScraperInputStream(fis,"UTF-8",new DigestURI("http://localhost"),null,false);
                                final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
                                if(charset != null)
                                    mimeType = mimeType + "; charset="+charset;
                                // reset position
                                fis.reset();
                            }
View Full Code Here

Examples of net.yacy.document.parser.html.ScraperInputStream.detectCharset()

162
163
164
165
166
167
168
169
170
171
172
        // nothing found: try to find a meta-tag
        if (charset == null) {
            try {
                final ScraperInputStream htmlFilter = new ScraperInputStream(sourceStream,documentCharset,location,null,false);
                sourceStream = htmlFilter;
                charset = htmlFilter.detectCharset();
            } catch (final IOException e1) {
                throw new Parser.Failure("Charset error:" + e1.getMessage(), location);
            }
        }
View Full Code Here

Examples of org.vietspider.html.util.HTMLParserDetector.detectCharset()

80
81
82
83
84
85
86
87
88
89
90
    if(level == depth) return ;
    System.out.println("\nstart download "+ address +" level "+level +" depth "+depth +" ...");
    byte [] bytes = download(parent.toString(), address);
    if(bytes == null || bytes.length < 0) return;
    HTMLParserDetector parser = new HTMLParserDetector();
    String charset = parser.detectCharset(bytes);
    char [] chars = CharsDecoder.decode(charset, bytes, 0, bytes.length);

    List<NodeImpl> tokens = parser.createTokens(chars);

    List<Resource> resources = new ArrayList<Resource>();
View Full Code Here

Examples of org.vietspider.html.util.HTMLParserDetector.detectCharset()

116
117
118
119
120
121
122
123
124
125
126
      message = "Error: Not html data!";
      return ERROR;
    }
    if(charset == null) {
      HTMLParserDetector detector = new HTMLParserDetector();
      charset = detector.detectCharset(data);
    }
   
    char [] chars = CharsDecoder.decode(charset, data, 0, data.length);
    return post(referer, formName, url, chars, abort);
  }
View Full Code Here

Examples of org.vietspider.html.util.HTMLParserDetector.detectCharset()

321
322
323
324
325
326
327
328
329
330
331
  }
 
  private boolean checkTimeout(byte [] data) throws Exception {
    if(data == null) return true;
    HTMLParserDetector parser = new HTMLParserDetector();
    if(charset == null) charset = parser.detectCharset(data);
   
    char [] chars = CharsDecoder.decode(charset, data, 0, data.length);
    List<NodeImpl> tokens  = parser.createTokens(chars);
    if(tokens == null) return true;
View Full Code Here

Examples of org.vietspider.html.util.HTMLParserDetector.detectCharset()

99
100
101
102
103
104
105
106
107
108
109
      message = "Not html data!";
      return ERROR;
    }
    if(charset == null) {
      HTMLParserDetector detector = new HTMLParserDetector();
      charset = detector.detectCharset(data);
    }
   
    char [] chars = CharsDecoder.decode(charset, data, 0, data.length);
    return post(referer, url, chars);
  }
View Full Code Here

Examples of org.vietspider.html.util.HTMLParserDetector.detectCharset()

82
83
84
85
86
87
88
89
90
91
92
    byte [] data = responseReader.readBody(response);
   
    HTMLParserDetector htmlParser2 = new HTMLParserDetector();
    if(data == null) return false;
    if(charset == null) {
      charset = htmlParser2.detectCharset(data);
    }
   
    char [] chars = CharsDecoder.decode(charset, data, 0, data.length);
    List<NodeImpl> tokens  = htmlParser2.createTokens(chars);
    if(tokens == null) return false;
View Full Code Here

Examples of org.vietspider.html.util.HTMLParserDetector.detectCharset()

80
81
82
83
84
85
86
87
88
89
90
    if(level == depth) return ;
    System.out.println("\nstart download "+ address +" level "+level +" depth "+depth +" ...");
    byte [] bytes = download(parent.toString(), address);
    if(bytes == null || bytes.length < 0) return;
    HTMLParserDetector parser = new HTMLParserDetector();
    String charset = parser.detectCharset(bytes);
    char [] chars = CharsDecoder.decode(charset, bytes, 0, bytes.length);

    List<NodeImpl> tokens = parser.createTokens(chars);

    List<Resource> resources = new ArrayList<Resource>();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.