/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nutch.util;
// JDK imports
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
// Hadoop imports
import org.apache.hadoop.conf.Configuration;
// Tika imports
import org.apache.tika.Tika;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MimeType;
import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.mime.MimeTypesFactory;
// Slf4j logging imports
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
// imported for Javadoc
import org.apache.nutch.protocol.ProtocolOutput;
/**
* @author mattmann
* @since NUTCH-608
*
* <p>
* This is a facade class to insulate Nutch from its underlying Mime Type
* substrate library, <a href="http://incubator.apache.org/tika/">Apache Tika</a>.
* Any mime handling code should be placed in this utility class, and hidden
* from the Nutch classes that rely on it.
* </p>
*/
public final class MimeUtil {
private static final String SEPARATOR = ";";
/* our Tika mime type registry */
private MimeTypes mimeTypes;
/* the tika detectors */
private Tika tika;
/* whether or not magic should be employed or not */
private boolean mimeMagic;
/* our log stream */
private static final Logger LOG = LoggerFactory.getLogger(MimeUtil.class.getName());
public MimeUtil(Configuration conf) {
tika = new Tika();
ObjectCache objectCache = ObjectCache.get(conf);
MimeTypes mimeTypez = (MimeTypes) objectCache.getObject(MimeTypes.class
.getName());
if (mimeTypez == null) {
try {
String customMimeTypeFile = conf.get("mime.types.file");
if (customMimeTypeFile!=null && customMimeTypeFile.equals("")==false){
try {
mimeTypez = MimeTypesFactory.create(conf
.getConfResourceAsInputStream(customMimeTypeFile));
}
catch (Exception e){
LOG.error("Can't load mime.types.file : "+customMimeTypeFile+" using Tika's default");
}
}
if (mimeTypez==null)
mimeTypez = MimeTypes.getDefaultMimeTypes();
} catch (Exception e) {
LOG.error("Exception in MimeUtil "+e.getMessage());
throw new RuntimeException(e);
}
objectCache.setObject(MimeTypes.class.getName(), mimeTypez);
}
this.mimeTypes = mimeTypez;
this.mimeMagic = conf.getBoolean("mime.type.magic", true);
}
/**
* Cleans a {@link MimeType} name by removing out the actual {@link MimeType},
* from a string of the form:
*
* <pre>
* <primary type>/<sub type> ; < optional params
* </pre>
*
* @param origType
* The original mime type string to be cleaned.
* @return The primary type, and subtype, concatenated, e.g., the actual mime
* type.
*/
public static String cleanMimeType(String origType) {
if (origType == null)
return null;
// take the origType and split it on ';'
String[] tokenizedMimeType = origType.split(SEPARATOR);
if (tokenizedMimeType.length > 1) {
// there was a ';' in there, take the first value
return tokenizedMimeType[0];
} else {
// there wasn't a ';', so just return the orig type
return origType;
}
}
/**
* A facade interface to trying all the possible mime type resolution
* strategies available within Tika. First, the mime type provided in
* <code>typeName</code> is cleaned, with {@link #cleanMimeType(String)}.
* Then the cleaned mime type is looked up in the underlying Tika
* {@link MimeTypes} registry, by its cleaned name. If the {@link MimeType}
* is found, then that mime type is used, otherwise URL resolution is
* used to try and determine the mime type. However, if
* <code>mime.type.magic</code> is enabled in {@link NutchConfiguration},
* then mime type magic resolution is used to try and obtain a
* better-than-the-default approximation of the {@link MimeType}.
*
* @param typeName
* The original mime type, returned from a {@link ProtocolOutput}.
* @param url
* The given @see url, that Nutch was trying to crawl.
* @param data
* The byte data, returned from the crawl, if any.
* @return The correctly, automatically guessed {@link MimeType} name.
*/
public String autoResolveContentType(String typeName, String url, byte[] data) {
String retType = null;
MimeType type = null;
String cleanedMimeType = null;
cleanedMimeType = MimeUtil.cleanMimeType(typeName);
// first try to get the type from the cleaned type name
if (cleanedMimeType != null) {
try {
type = mimeTypes.forName(cleanedMimeType);
cleanedMimeType = type.getName();
} catch (MimeTypeException mte) {
// Seems to be a malformed mime type name...
cleanedMimeType = null;
}
}
// if returned null, or if it's the default type then try url resolution
if (type == null
|| (type != null && type.getName().equals(MimeTypes.OCTET_STREAM))) {
// If no mime-type header, or cannot find a corresponding registered
// mime-type, then guess a mime-type from the url pattern
try {
retType = tika.detect(url) != null ? tika.detect(url) : null;
} catch (Exception e) {
String message = "Problem loading default Tika configuration";
LOG.error(message, e);
throw new RuntimeException(e);
}
} else {
retType = type.getName();
}
// if magic is enabled use mime magic to guess if the mime type returned
// from the magic guess is different than the one that's already set so far
// if it is, and it's not the default mime type, then go with the mime type
// returned by the magic
if (this.mimeMagic) {
String magicType = null;
// pass URL (file name) and (cleansed) content type from protocol to Tika
Metadata tikaMeta = new Metadata();
tikaMeta.add(Metadata.RESOURCE_NAME_KEY, url);
tikaMeta.add(Metadata.CONTENT_TYPE,
(cleanedMimeType != null ? cleanedMimeType : typeName));
try {
InputStream stream = TikaInputStream.get(data);
try {
magicType = tika.detect(stream, tikaMeta);
} finally {
stream.close();
}
} catch (IOException ignore) {}
if (magicType != null && !magicType.equals(MimeTypes.OCTET_STREAM)
&& !magicType.equals(MimeTypes.PLAIN_TEXT)
&& retType != null && !retType.equals(magicType)) {
// If magic enabled and the current mime type differs from that of the
// one returned from the magic, take the magic mimeType
retType = magicType;
}
// if type is STILL null after all the resolution strategies, go for the
// default type
if (retType == null) {
try {
retType = MimeTypes.OCTET_STREAM;
} catch (Exception ignore) {
}
}
}
return retType;
}
/**
* Facade interface to Tika's underlying {@link MimeTypes#getMimeType(String)}
* method.
*
* @param url
* A string representation of the document {@link URL} to sense the
* {@link MimeType} for.
* @return An appropriate {@link MimeType}, identified from the given
* Document url in string form.
*/
public String getMimeType(String url) {
return tika.detect(url);
}
/**
* A facade interface to Tika's underlying {@link MimeTypes#forName(String)}
* method.
*
* @param name
* The name of a valid {@link MimeType} in the Tika mime registry.
* @return The object representation of the {@link MimeType}, if it exists,
* or null otherwise.
*/
public String forName(String name) {
try {
return this.mimeTypes.forName(name).toString();
} catch (MimeTypeException e) {
LOG.error("Exception getting mime type by name: [" + name
+ "]: Message: " + e.getMessage());
return null;
}
}
/**
* Facade interface to Tika's underlying {@link MimeTypes#getMimeType(File)}
* method.
*
* @param f
* The {@link File} to sense the {@link MimeType} for.
* @return The {@link MimeType} of the given {@link File}, or null if it
* cannot be determined.
*/
public String getMimeType(File f) {
try {
return tika.detect(f);
} catch (Exception e) {
LOG.error("Exception getting mime type for file: [" + f.getPath()
+ "]: Message: " + e.getMessage());
return null;
}
}
}