Package org.apache.tika.parser.html

Examples of org.apache.tika.parser.html.HtmlParser

57
58
59
60
61
62
63
64
65
66
67
    super(UrlDatum.FIELDS);
  }

  private synchronized void init() {
    if (_parser == null) {
      _parser = new HtmlParser();
    }
   
    if (_handler == null) {
      _handler = new DefaultHandler() {
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
            fetcherPolicy.setFetcherMode(FetcherMode.EFFICIENT);
           
            // We only care about mime types that the Tika HTML parser can handle,
            // so restrict it to the same.
            Set<String> validMimeTypes = new HashSet<String>();
            Set<MediaType> supportedTypes = new HtmlParser().getSupportedTypes(new ParseContext());
            for (MediaType supportedType : supportedTypes) {
                validMimeTypes.add(String.format("%s/%s", supportedType.getType(), supportedType.getSubtype()));
            }
            fetcherPolicy.setValidMimeTypes(validMimeTypes);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    InputStream input = new ByteArrayInputStream(html.getBytes(Charset.forName("UTF-8")));
    ContentHandler text = new BodyContentHandler();//<co id="html.text.co"/>
    LinkContentHandler links = new LinkContentHandler();//<co id="html.link.co"/>
    ContentHandler handler = new TeeContentHandler(links, text);//<co id="html.merge"/>
    Metadata metadata = new Metadata();//<co id="html.store"/>
    Parser parser = new HtmlParser();//<co id="html.parser"/>
    ParseContext context = new ParseContext();
    parser.parse(input, handler, metadata, context);//<co id="html.parse"/>
    System.out.println("Title: " + metadata.get(Metadata.TITLE));
    System.out.println("Body: " + text.toString());
    System.out.println("Links: " + links.getLinks());
    /*
    <calloutlist>
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
     */
    private String extract(byte[] byteObject) throws TikaException {// throws IOException
        StringBuilder wBuf = new StringBuilder();
        InputStream stream = null;
        Metadata metadata = new Metadata();
        HtmlParser htmlParser = new HtmlParser();
        BodyContentHandler handler = new BodyContentHandler(-1);// -1
        ParseContext parser = new ParseContext();
        try {
            stream = new ByteArrayInputStream(byteObject);
            htmlParser.parse(stream, handler, metadata, parser);
            wBuf.append(handler.toString()
                    + System.getProperty("line.separator"));
        } catch (SAXException e) {
            throw new RuntimeException(e);
        } catch (IOException e) {
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
  @SuppressWarnings("serial")
  public static AutoDetectParser createParser() {
    final AutoDetectParser parser = new AutoDetectParser();

    Map<MediaType,Parser> parsers = parser.getParsers();
    parsers.put(MediaType.APPLICATION_XML, new HtmlParser());
    parser.setParsers(parsers);

    parser.setFallback(new Parser() {
      public Set<MediaType> getSupportedTypes(ParseContext parseContext) {
        return parser.getSupportedTypes(parseContext);
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
        StringTokenizer tokenizer = new StringTokenizer(classes, ", \t\n\r\f");
        while (tokenizer.hasMoreTokens()) {
            String name = tokenizer.nextToken();
            if (name.equals(
                    "org.apache.jackrabbit.extractor.HTMLTextExtractor")) {
                parsers.put(MediaType.text("html"), new HtmlParser());
            } else if (name.equals("org.apache.jackrabbit.extractor.MsExcelTextExtractor")) {
                Parser parser = new OfficeParser();
                parsers.put(MediaType.application("vnd.ms-excel"), parser);
                parsers.put(MediaType.application("msexcel"), parser);
                parsers.put(MediaType.application("excel"), parser);
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
        StringTokenizer tokenizer = new StringTokenizer(classes, ", \t\n\r\f");
        while (tokenizer.hasMoreTokens()) {
            String name = tokenizer.nextToken();
            if (name.equals(
                    "org.apache.jackrabbit.extractor.HTMLTextExtractor")) {
                parsers.put("text/html", new HtmlParser());
            } else if (name.equals(
                    "org.apache.jackrabbit.extractor.MsExcelTextExtractor")) {
                Parser parser = new OfficeParser();
                parsers.put("application/vnd.ms-excel", parser);
                parsers.put("application/msexcel", parser);
View Full Code Here
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
      Multipart mp = (Multipart) p.getContent();
      int count = mp.getCount();
      for (int i = 0; i < count; i++)
        content.append(getContentFromHTML(mp.getBodyPart(i)));
    } else if (p.isMimeType("text/html")) {
      HtmlParser parser = new HtmlParser();
      Metadata met = new Metadata();
      TextContentHandler handler = new TextContentHandler(
          new BodyContentHandler());
      parser.parse(new ByteArrayInputStream(((String) p.getContent())
          .getBytes()), handler, met);
      content.append(handler.toString());
    } else {
      Object obj = p.getContent();
      if (obj instanceof Part)
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
        // Read in link
        StringBuffer sb = new StringBuffer("");
        while (scanner.hasNext())
          sb.append(scanner.nextLine());

        HtmlParser parser = new HtmlParser();
        Metadata met = new Metadata();
        LinkContentHandler handler = new LinkContentHandler();

        parser.parse(new ByteArrayInputStream(sb.toString().getBytes()),
            handler, met);
        List<Link> links = handler.getLinks();
        children = new LinkedList<ProtocolFile>();
        for (Link link : links) {
          String href = link.getUri();
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
  public static AutoDetectParser createParser() {
    final AutoDetectParser parser = new AutoDetectParser();

    Map<MediaType,Parser> parsers = parser.getParsers();
    parsers.put(MediaType.APPLICATION_XML, new HtmlParser());
    parser.setParsers(parsers);

    parser.setFallback(new Parser() {
      public Set<MediaType> getSupportedTypes(ParseContext parseContext) {
        return parser.getSupportedTypes(parseContext);
View Full Code Here
TOP

Related Classes of org.apache.tika.parser.html.HtmlParser

  • bixo.examples.webmining.DemoWebMiningTool
  • com.scaleunlimited.helpful.operations.ParseModMboxPageFunction
  • com.tamingtext.tika.TikaTest
  • edu.uci.ics.crawler4j.parser.Parser
  • org.apache.commons.io.input.CloseShieldInputStream
  • org.apache.jackrabbit.core.query.lucene.JackrabbitParser
  • org.apache.oodt.cas.protocol.imaps.ImapsProtocol
  • org.apache.oodt.cas.pushpull.protocol.http.HttpClient
  • org.apache.oodt.cas.pushpull.protocol.imaps.ImapsClient
  • org.apache.tika.detect.AutoDetectReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.