/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.parser.microsoft;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.poi.hwpf.HWPFDocument;
import org.apache.poi.hwpf.HWPFOldDocument;
import org.apache.poi.hwpf.OldWordFileFormatException;
import org.apache.poi.hwpf.extractor.Word6Extractor;
import org.apache.poi.hwpf.model.FieldsDocumentPart;
import org.apache.poi.hwpf.model.PicturesTable;
import org.apache.poi.hwpf.model.StyleDescription;
import org.apache.poi.hwpf.usermodel.CharacterRun;
import org.apache.poi.hwpf.usermodel.Field;
import org.apache.poi.hwpf.usermodel.HeaderStories;
import org.apache.poi.hwpf.usermodel.Paragraph;
import org.apache.poi.hwpf.usermodel.Picture;
import org.apache.poi.hwpf.usermodel.Range;
import org.apache.poi.hwpf.usermodel.Table;
import org.apache.poi.hwpf.usermodel.TableCell;
import org.apache.poi.hwpf.usermodel.TableRow;
import org.apache.poi.poifs.filesystem.DirectoryEntry;
import org.apache.poi.poifs.filesystem.DirectoryNode;
import org.apache.poi.poifs.filesystem.Entry;
import org.apache.poi.poifs.filesystem.NPOIFSFileSystem;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
public class WordExtractor extends AbstractPOIFSExtractor {
private static final char UNICODECHAR_NONBREAKING_HYPHEN = '\u2011';
private static final char UNICODECHAR_ZERO_WIDTH_SPACE = '\u200b';
public WordExtractor(ParseContext context) {
super(context);
}
// True if we are currently in the named style tag:
private boolean curStrikeThrough;
private boolean curBold;
private boolean curItalic;
protected void parse(
NPOIFSFileSystem filesystem, XHTMLContentHandler xhtml)
throws IOException, SAXException, TikaException {
parse(filesystem.getRoot(), xhtml);
}
protected void parse(
DirectoryNode root, XHTMLContentHandler xhtml)
throws IOException, SAXException, TikaException {
HWPFDocument document;
try {
document = new HWPFDocument(root);
} catch(OldWordFileFormatException e) {
parseWord6(root, xhtml);
return;
}
org.apache.poi.hwpf.extractor.WordExtractor wordExtractor =
new org.apache.poi.hwpf.extractor.WordExtractor(document);
HeaderStories headerFooter = new HeaderStories(document);
// Grab the list of pictures. As far as we can tell,
// the pictures should be in order, and may be directly
// placed or referenced from an anchor
PicturesTable pictureTable = document.getPicturesTable();
PicturesSource pictures = new PicturesSource(document);
// Do any headers, if present
Range[] headers = new Range[] { headerFooter.getFirstHeaderSubrange(),
headerFooter.getEvenHeaderSubrange(), headerFooter.getOddHeaderSubrange() };
handleHeaderFooter(headers, "header", document, pictures, pictureTable, xhtml);
// Do the main paragraph text
Range r = document.getRange();
for(int i=0; i<r.numParagraphs(); i++) {
Paragraph p = r.getParagraph(i);
i += handleParagraph(p, 0, r, document, FieldsDocumentPart.MAIN, pictures, pictureTable, xhtml);
}
// Do everything else
for (String paragraph: wordExtractor.getMainTextboxText()) {
xhtml.element("p", paragraph);
}
for (String paragraph : wordExtractor.getFootnoteText()) {
xhtml.element("p", paragraph);
}
for (String paragraph : wordExtractor.getCommentsText()) {
xhtml.element("p", paragraph);
}
for (String paragraph : wordExtractor.getEndnoteText()) {
xhtml.element("p", paragraph);
}
// Do any footers, if present
Range[] footers = new Range[] { headerFooter.getFirstFooterSubrange(),
headerFooter.getEvenFooterSubrange(), headerFooter.getOddFooterSubrange() };
handleHeaderFooter(footers, "footer", document, pictures, pictureTable, xhtml);
// Handle any pictures that we haven't output yet
for(Picture p = pictures.nextUnclaimed(); p != null; ) {
handlePictureCharacterRun(
null, p, pictures, xhtml
);
p = pictures.nextUnclaimed();
}
// Handle any embeded office documents
try {
DirectoryEntry op = (DirectoryEntry) root.getEntry("ObjectPool");
for (Entry entry : op) {
if (entry.getName().startsWith("_")
&& entry instanceof DirectoryEntry) {
handleEmbeddedOfficeDoc((DirectoryEntry) entry, xhtml);
}
}
} catch(FileNotFoundException e) {
}
}
private static int countParagraphs(Range... ranges) {
int count = 0;
for (Range r : ranges) {
if (r != null) { count += r.numParagraphs(); }
}
return count;
}
private void handleHeaderFooter(Range[] ranges, String type, HWPFDocument document,
PicturesSource pictures, PicturesTable pictureTable, XHTMLContentHandler xhtml)
throws SAXException, IOException, TikaException {
if (countParagraphs(ranges) > 0) {
xhtml.startElement("div", "class", type);
for (Range r : ranges) {
if (r != null) {
for(int i=0; i<r.numParagraphs(); i++) {
Paragraph p = r.getParagraph(i);
String text = p.text();
if (text.replaceAll("[\\r\\n\\s]+", "").isEmpty()) {
// Skip empty header or footer paragraphs
} else {
i += handleParagraph(p, 0, r, document,
FieldsDocumentPart.HEADER, pictures, pictureTable, xhtml);
}
}
}
}
xhtml.endElement("div");
}
}
private int handleParagraph(Paragraph p, int parentTableLevel, Range r, HWPFDocument document,
FieldsDocumentPart docPart, PicturesSource pictures, PicturesTable pictureTable,
XHTMLContentHandler xhtml) throws SAXException, IOException, TikaException {
// Note - a poi bug means we can't currently properly recurse
// into nested tables, so currently we don't
if(p.isInTable() && p.getTableLevel() > parentTableLevel && parentTableLevel==0) {
Table t = r.getTable(p);
xhtml.startElement("table");
xhtml.startElement("tbody");
for(int rn=0; rn<t.numRows(); rn++) {
TableRow row = t.getRow(rn);
xhtml.startElement("tr");
for(int cn=0; cn<row.numCells(); cn++) {
TableCell cell = row.getCell(cn);
xhtml.startElement("td");
for(int pn=0; pn<cell.numParagraphs(); pn++) {
Paragraph cellP = cell.getParagraph(pn);
handleParagraph(cellP, p.getTableLevel(), cell, document, docPart, pictures, pictureTable, xhtml);
}
xhtml.endElement("td");
}
xhtml.endElement("tr");
}
xhtml.endElement("tbody");
xhtml.endElement("table");
return (t.numParagraphs()-1);
}
TagAndStyle tas;
if (document.getStyleSheet().numStyles()>p.getStyleIndex()) {
StyleDescription style =
document.getStyleSheet().getStyleDescription(p.getStyleIndex());
if (style != null && style.getName() != null && style.getName().length() > 0) {
tas = buildParagraphTagAndStyle(style.getName(), (parentTableLevel>0));
} else {
tas = new TagAndStyle("p", null);
}
} else {
tas = new TagAndStyle("p", null);
}
if(tas.getStyleClass() != null) {
xhtml.startElement(tas.getTag(), "class", tas.getStyleClass());
} else {
xhtml.startElement(tas.getTag());
}
for(int j=0; j<p.numCharacterRuns(); j++) {
CharacterRun cr = p.getCharacterRun(j);
// FIELD_BEGIN_MARK:
if (cr.text().getBytes()[0] == 0x13) {
Field field = document.getFields().getFieldByStartOffset(docPart, cr.getStartOffset());
// 58 is an embedded document
// 56 is a document link
if (field != null && (field.getType() == 58 || field.getType() == 56)) {
// Embedded Object: add a <div
// class="embedded" id="_X"/> so consumer can see where
// in the main text each embedded document
// occurred:
String id = "_" + field.getMarkSeparatorCharacterRun(r).getPicOffset();
AttributesImpl attributes = new AttributesImpl();
attributes.addAttribute("", "class", "class", "CDATA", "embedded");
attributes.addAttribute("", "id", "id", "CDATA", id);
xhtml.startElement("div", attributes);
xhtml.endElement("div");
}
}
if(cr.text().equals("\u0013")) {
j += handleSpecialCharacterRuns(p, j, tas.isHeading(), pictures, xhtml);
} else if(cr.text().startsWith("\u0008")) {
// Floating Picture(s)
for(int pn=0; pn<cr.text().length(); pn++) {
// Assume they're in the order from the unclaimed list...
Picture picture = pictures.nextUnclaimed();
// Output
handlePictureCharacterRun(cr, picture, pictures, xhtml);
}
} else if(pictureTable.hasPicture(cr)) {
// Inline Picture
Picture picture = pictures.getFor(cr);
handlePictureCharacterRun(cr, picture, pictures, xhtml);
} else {
handleCharacterRun(cr, tas.isHeading(), xhtml);
}
}
// Close any still open style tags
if (curStrikeThrough) {
xhtml.endElement("s");
curStrikeThrough = false;
}
if (curItalic) {
xhtml.endElement("i");
curItalic = false;
}
if (curBold) {
xhtml.endElement("b");
curBold = false;
}
xhtml.endElement(tas.getTag());
return 0;
}
private void handleCharacterRun(CharacterRun cr, boolean skipStyling, XHTMLContentHandler xhtml)
throws SAXException {
// Skip trailing newlines
if(!isRendered(cr) || cr.text().equals("\r"))
return;
if(!skipStyling) {
if (cr.isBold() != curBold) {
// Enforce nesting -- must close s and i tags
if (curStrikeThrough) {
xhtml.endElement("s");
curStrikeThrough = false;
}
if (curItalic) {
xhtml.endElement("i");
curItalic = false;
}
if (cr.isBold()) {
xhtml.startElement("b");
} else {
xhtml.endElement("b");
}
curBold = cr.isBold();
}
if (cr.isItalic() != curItalic) {
// Enforce nesting -- must close s tag
if (curStrikeThrough) {
xhtml.endElement("s");
curStrikeThrough = false;
}
if (cr.isItalic()) {
xhtml.startElement("i");
} else {
xhtml.endElement("i");
}
curItalic = cr.isItalic();
}
if (cr.isStrikeThrough() != curStrikeThrough) {
if (cr.isStrikeThrough()) {
xhtml.startElement("s");
} else {
xhtml.endElement("s");
}
curStrikeThrough = cr.isStrikeThrough();
}
}
// Clean up the text
String text = cr.text();
text = text.replace('\r', '\n');
if(text.endsWith("\u0007")) {
// Strip the table cell end marker
text = text.substring(0, text.length()-1);
}
// Copied from POI's org/apache/poi/hwpf/converter/AbstractWordConverter.processCharacters:
// line tabulator as break line
text = text.replace((char)0x000b,'\n');
// Non-breaking hyphens are returned as char 30
text = text.replace((char) 30, UNICODECHAR_NONBREAKING_HYPHEN);
// Non-required hyphens to zero-width space
text = text.replace((char) 31, UNICODECHAR_ZERO_WIDTH_SPACE);
xhtml.characters(text);
}
/**
* Can be \13..text..\15 or \13..control..\14..text..\15 .
* Nesting is allowed
*/
private int handleSpecialCharacterRuns(Paragraph p, int index, boolean skipStyling,
PicturesSource pictures, XHTMLContentHandler xhtml) throws SAXException, TikaException, IOException {
List<CharacterRun> controls = new ArrayList<CharacterRun>();
List<CharacterRun> texts = new ArrayList<CharacterRun>();
boolean has14 = false;
// Split it into before and after the 14
int i;
for(i=index+1; i<p.numCharacterRuns(); i++) {
CharacterRun cr = p.getCharacterRun(i);
if(cr.text().equals("\u0013")) {
// Nested, oh joy...
int increment = handleSpecialCharacterRuns(p, i+1, skipStyling, pictures, xhtml);
i += increment;
} else if(cr.text().equals("\u0014")) {
has14 = true;
} else if(cr.text().equals("\u0015")) {
if(!has14) {
texts = controls;
controls = new ArrayList<CharacterRun>();
}
break;
} else {
if(has14) {
texts.add(cr);
} else {
controls.add(cr);
}
}
}
// Do we need to do something special with this?
if(controls.size() > 0) {
String text = controls.get(0).text();
for(int j=1; j<controls.size(); j++) {
text += controls.get(j).text();
}
if((text.startsWith("HYPERLINK") || text.startsWith(" HYPERLINK"))
&& text.indexOf('"') > -1) {
String url = text.substring(
text.indexOf('"') + 1,
text.lastIndexOf('"')
);
xhtml.startElement("a", "href", url);
for(CharacterRun cr : texts) {
handleCharacterRun(cr, skipStyling, xhtml);
}
xhtml.endElement("a");
} else {
// Just output the text ones
for(CharacterRun cr : texts) {
if(pictures.hasPicture(cr)) {
Picture picture = pictures.getFor(cr);
handlePictureCharacterRun(cr, picture, pictures, xhtml);
} else {
handleCharacterRun(cr, skipStyling, xhtml);
}
}
}
} else {
// We only had text
// Output as-is
for(CharacterRun cr : texts) {
handleCharacterRun(cr, skipStyling, xhtml);
}
}
// Tell them how many to skip over
return i-index;
}
private void handlePictureCharacterRun(CharacterRun cr, Picture picture, PicturesSource pictures, XHTMLContentHandler xhtml)
throws SAXException, IOException, TikaException {
if(!isRendered(cr) || picture == null) {
// Oh dear, we've run out...
// Probably caused by multiple \u0008 images referencing
// the same real image
return;
}
// Which one is it?
String extension = picture.suggestFileExtension();
int pictureNumber = pictures.pictureNumber(picture);
// Make up a name for the picture
// There isn't one in the file, but we need to be able to reference
// the picture from the img tag and the embedded resource
String filename = "image"+pictureNumber+(extension.length()>0 ? "."+extension : "");
// Grab the mime type for the picture
String mimeType = picture.getMimeType();
// Output the img tag
AttributesImpl attr = new AttributesImpl();
attr.addAttribute("", "src", "src", "CDATA", "embedded:" + filename);
attr.addAttribute("", "alt", "alt", "CDATA", filename);
xhtml.startElement("img", attr);
xhtml.endElement("img");
// Have we already output this one?
// (Only expose each individual image once)
if(! pictures.hasOutput(picture)) {
TikaInputStream stream = TikaInputStream.get(picture.getContent());
handleEmbeddedResource(stream, filename, null, mimeType, xhtml, false);
pictures.recordOutput(picture);
}
}
/**
* Outputs a section of text if the given text is non-empty.
*
* @param xhtml XHTML content handler
* @param section the class of the <div/> section emitted
* @param text text to be emitted, if any
* @throws SAXException if an error occurs
*/
private void addTextIfAny(
XHTMLContentHandler xhtml, String section, String text)
throws SAXException {
if (text != null && text.length() > 0) {
xhtml.startElement("div", "class", section);
xhtml.element("p", text);
xhtml.endElement("div");
}
}
protected void parseWord6(
NPOIFSFileSystem filesystem, XHTMLContentHandler xhtml)
throws IOException, SAXException, TikaException {
parseWord6(filesystem.getRoot(), xhtml);
}
protected void parseWord6(
DirectoryNode root, XHTMLContentHandler xhtml)
throws IOException, SAXException, TikaException {
HWPFOldDocument doc = new HWPFOldDocument(root);
Word6Extractor extractor = new Word6Extractor(doc);
for(String p : extractor.getParagraphText()) {
xhtml.element("p", p);
}
}
private static final Map<String,TagAndStyle> fixedParagraphStyles = new HashMap<String,TagAndStyle>();
private static final TagAndStyle defaultParagraphStyle = new TagAndStyle("p", null);
static {
fixedParagraphStyles.put("Default", defaultParagraphStyle);
fixedParagraphStyles.put("Normal", defaultParagraphStyle);
fixedParagraphStyles.put("heading", new TagAndStyle("h1", null));
fixedParagraphStyles.put("Heading", new TagAndStyle("h1", null));
fixedParagraphStyles.put("Title", new TagAndStyle("h1", "title"));
fixedParagraphStyles.put("Subtitle", new TagAndStyle("h2", "subtitle"));
fixedParagraphStyles.put("HTML Preformatted", new TagAndStyle("pre", null));
}
/**
* Given a style name, return what tag should be used, and
* what style should be applied to it.
*/
public static TagAndStyle buildParagraphTagAndStyle(String styleName, boolean isTable) {
TagAndStyle tagAndStyle = fixedParagraphStyles.get(styleName);
if (tagAndStyle != null) {
return tagAndStyle;
}
if (styleName.equals("Table Contents") && isTable) {
return defaultParagraphStyle;
}
String tag = "p";
String styleClass = null;
if(styleName.startsWith("heading") || styleName.startsWith("Heading")) {
// "Heading 3" or "Heading2" or "heading 4"
int num = 1;
try {
num = Integer.parseInt(
styleName.substring(styleName.length()-1)
);
} catch(NumberFormatException e) {}
// Turn it into a H1 - H6 (H7+ isn't valid!)
tag = "h" + Math.min(num, 6);
} else {
styleClass = styleName.replace(' ', '_');
styleClass = styleClass.substring(0,1).toLowerCase() +
styleClass.substring(1);
}
return new TagAndStyle(tag,styleClass);
}
public static class TagAndStyle {
private String tag;
private String styleClass;
public TagAndStyle(String tag, String styleClass) {
this.tag = tag;
this.styleClass = styleClass;
}
public String getTag() {
return tag;
}
public String getStyleClass() {
return styleClass;
}
public boolean isHeading() {
return tag.length()==2 && tag.startsWith("h");
}
}
/**
* Determines if character run should be included in the extraction.
*
* @param cr character run.
* @return true if character run should be included in extraction.
*/
private boolean isRendered(final CharacterRun cr) {
return cr == null || !cr.isMarkedDeleted();
}
/**
* Provides access to the pictures both by offset, iteration
* over the un-claimed, and peeking forward
*/
private static class PicturesSource {
private PicturesTable picturesTable;
private Set<Picture> output = new HashSet<Picture>();
private Map<Integer,Picture> lookup;
private List<Picture> nonU1based;
private List<Picture> all;
private int pn = 0;
private PicturesSource(HWPFDocument doc) {
picturesTable = doc.getPicturesTable();
all = picturesTable.getAllPictures();
// Build the Offset-Picture lookup map
lookup = new HashMap<Integer, Picture>();
for(Picture p : all) {
lookup.put(p.getStartOffset(), p);
}
// Work out which Pictures aren't referenced by
// a \u0001 in the main text
// These are \u0008 escher floating ones, ones
// found outside the normal text, and who
// knows what else...
nonU1based = new ArrayList<Picture>();
nonU1based.addAll(all);
Range r = doc.getRange();
for(int i=0; i<r.numCharacterRuns(); i++) {
CharacterRun cr = r.getCharacterRun(i);
if(picturesTable.hasPicture(cr)) {
Picture p = getFor(cr);
int at = nonU1based.indexOf(p);
nonU1based.set(at, null);
}
}
}
private boolean hasPicture(CharacterRun cr) {
return picturesTable.hasPicture(cr);
}
private void recordOutput(Picture picture) {
output.add(picture);
}
private boolean hasOutput(Picture picture) {
return output.contains(picture);
}
private int pictureNumber(Picture picture) {
return all.indexOf(picture) + 1;
}
private Picture getFor(CharacterRun cr) {
return lookup.get(cr.getPicOffset());
}
/**
* Return the next unclaimed one, used towards
* the end
*/
private Picture nextUnclaimed() {
Picture p = null;
while(pn < nonU1based.size()) {
p = nonU1based.get(pn);
pn++;
if(p != null) return p;
}
return null;
}
}
}