/* The contents of this file are subject to the license and copyright terms
* detailed in the license directory at the root of the source tree (also
* available online at http://fedora-commons.org/license/).
*/
package org.fcrepo.server.storage.translation;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.xml.stream.XMLStreamException;
import javax.activation.MimeType;
import org.apache.commons.io.IOUtils;
import org.apache.abdera.Abdera;
import org.apache.abdera.ext.thread.ThreadHelper;
import org.apache.abdera.i18n.iri.IRI;
import org.apache.abdera.model.Category;
import org.apache.abdera.model.Content;
import org.apache.abdera.model.Document;
import org.apache.abdera.model.Entry;
import org.apache.abdera.model.Feed;
import org.apache.abdera.model.Person;
import org.apache.abdera.parser.Parser;
import org.apache.abdera.util.MimeTypeHelper;
import org.apache.abdera.xpath.XPath;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.fcrepo.common.Constants;
import org.fcrepo.common.MalformedPIDException;
import org.fcrepo.common.PID;
import org.fcrepo.common.xml.format.XMLFormat;
import org.fcrepo.server.errors.ObjectIntegrityException;
import org.fcrepo.server.errors.StreamIOException;
import org.fcrepo.server.errors.ValidationException;
import org.fcrepo.server.storage.types.Datastream;
import org.fcrepo.server.storage.types.DatastreamManagedContent;
import org.fcrepo.server.storage.types.DatastreamReferencedContent;
import org.fcrepo.server.storage.types.DatastreamXMLMetadata;
import org.fcrepo.server.storage.types.DigitalObject;
import org.fcrepo.server.validation.ValidationUtility;
import org.fcrepo.utilities.DateUtility;
import org.fcrepo.utilities.FileUtils;
import org.fcrepo.utilities.NormalizedURI;
/**
* Deserializer for Fedora Objects in Atom format.
*
* @author Edwin Shin
* @since 3.0
* @version $Id$
*/
public class AtomDODeserializer
implements DODeserializer, Constants {
public static final XMLFormat DEFAULT_FORMAT = ATOM1_1;
private static final Logger logger =
LoggerFactory.getLogger(AtomDODeserializer.class);
/** The object to deserialize to. */
private DigitalObject m_obj;
private String m_encoding;
/** The current translation context. */
private int m_transContext;
/** The format this deserializer reads. */
private final XMLFormat m_format;
private final Abdera abdera = Abdera.getInstance();
private Feed m_feed;
private XPath m_xpath;
private ZipInputStream m_zin;
/**
* Temporary directory for the unpacked contents of an Atom Zip archive.
*/
private File m_tempDir;
public AtomDODeserializer() {
this(DEFAULT_FORMAT);
}
public AtomDODeserializer(XMLFormat format) {
if (format.equals(ATOM1_1) || format.equals(ATOM_ZIP1_1)) {
m_format = format;
} else {
throw new IllegalArgumentException("Not an Atom format: "
+ format.uri);
}
}
/**
* {@inheritDoc}
*/
public void deserialize(InputStream in,
DigitalObject obj,
String encoding,
int transContext) throws ObjectIntegrityException,
StreamIOException, UnsupportedEncodingException {
if (m_format.equals(ATOM_ZIP1_1)) {
try {
m_tempDir = FileUtils.createTempDir("atomzip", null);
m_zin = new ZipInputStream(new BufferedInputStream(in));
ZipEntry entry;
while ((entry = m_zin.getNextEntry()) != null) {
FileUtils.copy(m_zin, new FileOutputStream(new File(m_tempDir, entry.getName())));
}
in = new FileInputStream(new File(m_tempDir, "atommanifest.xml"));
} catch (FileNotFoundException e) {
throw new StreamIOException(e.getMessage(), e);
} catch (IOException e) {
throw new StreamIOException(e.getMessage(), e);
}
}
Parser parser = abdera.getParser();
Document<Feed> feedDoc = parser.parse(in);
m_feed = feedDoc.getRoot();
m_xpath = abdera.getXPath();
m_obj = obj;
m_encoding = encoding;
m_transContext = transContext;
addObjectProperties();
addDatastreams();
DOTranslationUtility.normalizeDatastreams(m_obj,
m_transContext,
m_encoding);
FileUtils.delete(m_tempDir);
}
/**
* {@inheritDoc}
*/
public DODeserializer getInstance() {
return new AtomDODeserializer(m_format);
}
/**
* Set the Fedora Object properties from the Feed metadata.
*
* @throws ObjectIntegrityException
*/
private void addObjectProperties() throws ObjectIntegrityException {
PID pid;
try {
pid = new PID(m_feed.getId().toString());
} catch (MalformedPIDException e) {
throw new ObjectIntegrityException(e.getMessage(), e);
}
String label = m_feed.getTitle();
String state =
m_xpath.valueOf("/a:feed/a:category[@scheme='"
+ MODEL.STATE.uri + "']/@term", m_feed);
String createDate =
m_xpath.valueOf("/a:feed/a:category[@scheme='"
+ MODEL.CREATED_DATE.uri + "']/@term", m_feed);
m_obj.setPid(pid.toString());
try {
m_obj.setState(DOTranslationUtility.readStateAttribute(state));
} catch (ParseException e) {
throw new ObjectIntegrityException("Could not read object state", e);
}
m_obj.setLabel(label);
m_obj.setOwnerId(getOwnerId());
m_obj.setCreateDate(DateUtility.convertStringToDate(createDate));
m_obj.setLastModDate(m_feed.getUpdated());
setExtProps();
}
private void addDatastreams() throws UnsupportedEncodingException,
StreamIOException, ObjectIntegrityException {
m_feed.sortEntries(new UpdatedIdComparator(true));
List<Entry> entries = m_feed.getEntries();
for (Entry entry : entries) {
if (ThreadHelper.getInReplyTo(entry) != null) {
addDatastreamVersion(entry);
}
}
}
private void addDatastreamVersion(Entry entry)
throws UnsupportedEncodingException, StreamIOException,
ObjectIntegrityException {
IRI ref = ThreadHelper.getInReplyTo(entry).getRef();
Entry parent = m_feed.getEntry(ref.toString());
Datastream ds;
String controlGroup = getDSControlGroup(parent);
if (controlGroup.equals("X")) {
ds = addInlineDatastreamVersion(entry);
} else if (controlGroup.equals("M")) {
ds = addManagedDatastreamVersion(entry);
} else {
ds = addExternalReferencedDatastreamVersion(entry);
}
m_obj.addDatastreamVersion(ds, true);
}
private Datastream addInlineDatastreamVersion(Entry entry)
throws ObjectIntegrityException, StreamIOException {
DatastreamXMLMetadata ds = new DatastreamXMLMetadata();
setDSCommonProperties(ds, entry);
String dsId = ds.DatastreamID;
String dsvId = ds.DSVersionID;
ds.DSLocation = m_obj.getPid() + "+" + dsId + "+" + dsvId;
if (ds.DSVersionID.equals("AUDIT.0")) {
addAuditDatastream(entry);
} else {
try {
if (m_format.equals(ATOM_ZIP1_1)) {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
FileUtils.copy(new FileInputStream(getContentSrcAsFile(entry.getContentSrc())),
bout);
ds.xmlContent = bout.toByteArray();
} else {
ds.xmlContent = entry.getContent().getBytes(m_encoding); //IOUtils.toByteArray(entry.getContentStream());
}
} catch (UnsupportedEncodingException e) {
throw new StreamIOException(e.getMessage(), e);
} catch (FileNotFoundException e) {
throw new ObjectIntegrityException(e.getMessage(), e);
}
}
if (ds.xmlContent != null) {
ds.DSSize = ds.xmlContent.length;
}
MimeType mimeType = entry.getContentMimeType();
if (mimeType == null) {
ds.DSMIME = "text/xml";
} else {
ds.DSMIME = mimeType.toString();
}
return ds;
}
private Datastream addExternalReferencedDatastreamVersion(Entry entry)
throws ObjectIntegrityException {
Datastream ds = new DatastreamReferencedContent();
setDSCommonProperties(ds, entry);
ds.DSLocation = entry.getContentSrc().toString();
// Normalize the dsLocation for the deserialization context
ds.DSLocation =
(DOTranslationUtility.normalizeDSLocationURLs(m_obj.getPid(),
ds,
m_transContext)).DSLocation;
ds.DSLocationType = Datastream.DS_LOCATION_TYPE_URL;
ds.DSMIME = entry.getContentMimeType().toString();
return ds;
}
private Datastream addManagedDatastreamVersion(Entry entry)
throws StreamIOException, ObjectIntegrityException {
Datastream ds = new DatastreamManagedContent();
setDSCommonProperties(ds, entry);
ds.DSLocationType = Datastream.DS_LOCATION_TYPE_INTERNAL;
ds.DSMIME = getDSMimeType(entry);
// Managed Content can take any of the following forms:
// 1) inline text (plaintext, html, xml)
// 2) inline Base64
// 3) referenced content
IRI contentLocation = entry.getContentSrc();
if (contentLocation != null) {
// URL FORMAT VALIDATION for dsLocation:
// For Managed Content the URL is only checked when we are parsing a
// a NEW ingest file because the URL is replaced with an internal identifier
// once the repository has sucked in the content for storage.
// AtomZIP files can have a simple filename (nb, not file: )for the content location, so don't validate that
if (m_obj.isNew() && !m_format.equals(ATOM_ZIP1_1)) {
ValidationUtility
.validateURL(contentLocation.toString(),ds.DSControlGrp);
}
if (m_format.equals(ATOM_ZIP1_1)) {
if (!contentLocation.isAbsolute() && !contentLocation.isPathAbsolute()) {
File f = getContentSrcAsFile(contentLocation);
contentLocation = new IRI(DatastreamManagedContent.TEMP_SCHEME +
f.getAbsolutePath());
}
}
ds.DSLocation = contentLocation.toString();
ds.DSLocation =
(DOTranslationUtility.normalizeDSLocationURLs(m_obj
.getPid(), ds, m_transContext)).DSLocation;
return ds;
}
try {
File temp = File.createTempFile("binary-datastream", null);
OutputStream out = new FileOutputStream(temp);
if (MimeTypeHelper.isText(ds.DSMIME)
|| MimeTypeHelper.isXml(ds.DSMIME)) {
IOUtils.copy(new StringReader(entry.getContent()),
out,
m_encoding);
} else {
IOUtils.copy(entry.getContentStream(), out);
}
ds.DSLocation = DatastreamManagedContent.TEMP_SCHEME + temp.getAbsolutePath();
} catch (IOException e) {
throw new StreamIOException(e.getMessage(), e);
}
return ds;
}
private void addAuditDatastream(Entry entry)
throws ObjectIntegrityException, StreamIOException {
try {
Reader auditTrail;
if (m_format.equals(ATOM_ZIP1_1)) {
File f = getContentSrcAsFile(entry.getContentSrc());
auditTrail = new InputStreamReader(new FileInputStream(f), m_encoding);
} else {
auditTrail = new StringReader(entry.getContent());
}
m_obj.getAuditRecords().addAll(DOTranslationUtility
.getAuditRecords(auditTrail));
auditTrail.close();
} catch (XMLStreamException e) {
throw new ObjectIntegrityException(e.getMessage(), e);
} catch (IOException e) {
throw new StreamIOException(e.getMessage(), e);
}
}
private String getOwnerId() {
Person owner = m_feed.getAuthor();
if (owner == null) {
return "";
} else {
return owner.getName();
}
}
/**
* Parses the id to determine a datastreamId.
*
* @param id
* @return
*/
private String getDatastreamId(Entry entry) {
String entryId = entry.getId().toString();
// matches info:fedora/pid/dsid/timestamp
Pattern pattern =
Pattern.compile("^" + Constants.FEDORA.uri + ".+?/([^/]+)/?.*");
Matcher matcher = pattern.matcher(entryId);
if (matcher.find()) {
return matcher.group(1);
} else {
return m_obj.newDatastreamID();
}
}
private String getDatastreamVersionId(Entry entry) {
String dsId = getDatastreamId(entry);
String dsvId = entry.getTitle();
// e.g. Match DS1.0 but not DS1
if (dsvId.matches("^" + dsId + ".*\\.[\\w]")) {
return dsvId;
} else {
if (!m_obj.datastreams(dsId).iterator().hasNext()) {
return dsId + ".0";
} else {
return m_obj.newDatastreamID(dsId);
}
}
}
private String getDSControlGroup(Entry entry)
throws ObjectIntegrityException {
List<Category> controlGroups =
entry.getCategories(MODEL.CONTROL_GROUP.uri);
// Try to infer the control group if not provided
if (controlGroups.isEmpty() || controlGroups.size() > 1) {
if (entry.getContentType() != null) {
if (entry.getContentType().equals(Content.Type.XML)) {
return "X";
} else {
// only XML can be inline
return "M";
}
}
if (entry.getContentSrc() != null) {
return "M";
}
// TODO other cases
// link alts, link enclosures
else {
throw new ObjectIntegrityException("No control group provided by "
+ m_obj.getPid());
}
} else {
return controlGroups.get(0).getTerm();
}
}
private String getDSState(Entry entry) {
List<Category> state = entry.getCategories(MODEL.STATE.uri);
if (state.isEmpty() || state.size() > 1) {
return "A";
} else {
return state.get(0).getTerm();
}
}
/**
* Note: AUDIT datastreams always return false, otherwise defaults to true.
*
* @param entry
* @return
*/
private boolean getDSVersionable(Entry entry) {
if (getDatastreamId(entry).equals("AUDIT")) {
return false;
}
List<Category> versionable = entry.getCategories(MODEL.VERSIONABLE.uri);
if (versionable.isEmpty() || versionable.size() > 1) {
return true;
} else {
return Boolean.valueOf(versionable.get(0).getTerm());
}
}
private String[] getDSAltIds(Entry entry) {
List<Category> altIds = entry.getCategories(MODEL.ALT_IDS.uri);
if (altIds.isEmpty()) {
return new String[0];
} else {
return altIds.get(0).getTerm().split(" ");
// TODO we could handle size > 1
}
}
private String getDSFormatURI(Entry entry) {
List<Category> formatURI = entry.getCategories(MODEL.FORMAT_URI.uri);
if (formatURI.isEmpty() || formatURI.size() > 1) {
return null;
} else {
return formatURI.get(0).getTerm();
}
}
private String getDSLabel(Entry entry) {
List<Category> label = entry.getCategories(MODEL.LABEL.uri);
if (label.isEmpty()) {
return "";
}
return label.get(0).getTerm();
}
private String getDSMimeType(Entry entry) {
String dsMimeType = "application/unknown";
MimeType mimeType = entry.getContentMimeType();
if (mimeType == null) {
Content.Type type = entry.getContentType();
if (type != null) {
if (type == Content.Type.HTML) {
dsMimeType = "text/html";
} else if (type == Content.Type.TEXT) {
dsMimeType = "text/plain";
} else if (type == Content.Type.XHTML) {
dsMimeType = "application/xhtml+xml";
} else if (type == Content.Type.XML) {
dsMimeType = "text/xml";
}
}
} else {
dsMimeType = mimeType.toString();
}
return dsMimeType;
}
private String getDSChecksumType(Entry entry) {
List<Category> digestType = entry.getCategories(MODEL.DIGEST_TYPE.uri);
if (digestType.isEmpty()) {
String result = (Datastream.autoChecksum)
? Datastream.getDefaultChecksumType()
: Datastream.CHECKSUMTYPE_DISABLED;
return result;
} else {
return digestType.get(0).getTerm();
}
}
private String getDSChecksum(Entry entry) {
List<Category> digest = entry.getCategories(MODEL.DIGEST.uri);
if (digest.isEmpty()) {
return Datastream.CHECKSUM_NONE;
} else {
return digest.get(0).getTerm();
}
}
private void setDSCommonProperties(Datastream dsVersion, Entry entry)
throws ObjectIntegrityException {
IRI ref = ThreadHelper.getInReplyTo(entry).getRef();
Entry parent = m_feed.getEntry(ref.toString());
dsVersion.DatastreamID = getDatastreamId(parent);
dsVersion.DSControlGrp = getDSControlGroup(parent);
dsVersion.DSState = getDSState(parent);
dsVersion.DSVersionable = getDSVersionable(parent);
setDatastreamVersionProperties(dsVersion, entry);
}
private void setDatastreamVersionProperties(Datastream ds, Entry entry)
throws ValidationException {
ds.DatastreamAltIDs = getDSAltIds(entry);
ds.DSCreateDT = entry.getUpdated();
ds.DSFormatURI = getDSFormatURI(entry);
ds.DSLabel = getDSLabel(entry);
ds.DSVersionID = getDatastreamVersionId(entry);
ds.DSChecksumType = getDSChecksumType(entry);
String checksum = getDSChecksum(entry);
if (m_obj.isNew()) {
if (logger.isDebugEnabled()) {
logger.debug("New Object: checking supplied checksum");
}
if (checksum != null && !checksum.equals("")
&& !checksum.equals(Datastream.CHECKSUM_NONE)) {
String tmpChecksum = ds.getChecksum();
if (logger.isDebugEnabled()) {
logger.debug("checksum = " + tmpChecksum);
}
if (!checksum.equals(tmpChecksum)) {
throw new ValidationException("Checksum Mismatch: "
+ tmpChecksum);
}
}
ds.DSChecksumType = ds.getChecksumType();
} else {
ds.DSChecksum = checksum;
}
}
private void setExtProps() {
List<Category> epCategories =
m_feed.getCategories(MODEL.EXT_PROPERTY.uri);
for (Category epCategory : epCategories) {
m_obj.setExtProperty(epCategory.getTerm(), epCategory.getLabel());
}
}
/**
* Returns the an Entry's contentSrc as a File relative to {@link #m_tempDir}.
*
* @param contentSrc
* @return the contentSrc as a File relative to m_tempDir.
* @throws ObjectIntegrityException
*/
protected File getContentSrcAsFile(IRI contentSrc) throws ObjectIntegrityException {
if (contentSrc.isAbsolute() || contentSrc.isPathAbsolute()) {
throw new ObjectIntegrityException("contentSrc must not be absolute");
}
try {
// Normalize the IRI to resolve percent-encoding and
// backtracking (e.g. "../")
NormalizedURI nUri = new NormalizedURI(m_tempDir.toURI().toString() + contentSrc.toString());
nUri.normalize();
File f = new File(nUri.toURI());
if (f.getParentFile().equals(m_tempDir)) {
return f;
} else {
throw new ObjectIntegrityException(contentSrc.toString()
+ " is not a valid path.");
}
} catch (URISyntaxException e) {
throw new ObjectIntegrityException(e.getMessage(), e);
}
}
private static class UpdatedIdComparator
implements Comparator<Entry> {
private boolean ascending = true;
UpdatedIdComparator(boolean ascending) {
this.ascending = ascending;
}
public int compare(Entry o1, Entry o2) {
Date d1 = o1.getUpdated();
Date d2 = o2.getUpdated();
String id1 = o1.getId().toString();
String id2 = o2.getId().toString();
int r = d1.compareTo(d2);
if (d1.equals(d2)) {
r = id1.compareTo(id2);
}
return (ascending) ? r : -r;
}
}
}