package org.yaac.server.util;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Strings.isNullOrEmpty;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.yaac.server.egql.evaluator.EvaluationResult;
import org.yaac.server.egql.evaluator.Evaluator;
import org.yaac.server.egql.evaluator.function.BlobFunction.BlobFileRefWrapper;
import org.yaac.server.egql.evaluator.function.BlobFunction.BlobStringWrapper;
import org.yaac.server.egql.evaluator.function.TextFunction.TextFileRefWrapper;
import org.yaac.server.egql.evaluator.function.TextFunction.TextStringWrapper;
import org.yaac.shared.YaacException;
import org.yaac.shared.crud.MetaKind;
import org.yaac.shared.crud.MetaNamespace;
import org.yaac.shared.editor.EntityInfo;
import org.yaac.shared.file.FileDownloadPath;
import org.yaac.shared.property.BlobKeyPropertyInfo;
import org.yaac.shared.property.BlobPropertyInfo;
import org.yaac.shared.property.BooleanPropertyInfo;
import org.yaac.shared.property.DatePropertyInfo;
import org.yaac.shared.property.DoublePropertyInfo;
import org.yaac.shared.property.GeoPtPropertyInfo;
import org.yaac.shared.property.IMHandlePropertyInfo;
import org.yaac.shared.property.KeyInfo;
import org.yaac.shared.property.ListPropertyInfo;
import org.yaac.shared.property.LongPropertyInfo;
import org.yaac.shared.property.NullPropertyInfo;
import org.yaac.shared.property.PropertyInfo;
import org.yaac.shared.property.StringPropertyInfo;
import org.yaac.shared.property.TextPropertyInfo;
import org.yaac.shared.property.UserPropertyInfo;
import com.google.appengine.api.blobstore.BlobKey;
import com.google.appengine.api.datastore.Blob;
import com.google.appengine.api.datastore.Category;
import com.google.appengine.api.datastore.Email;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.GeoPt;
import com.google.appengine.api.datastore.IMHandle;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.api.datastore.Link;
import com.google.appengine.api.datastore.PhoneNumber;
import com.google.appengine.api.datastore.PostalAddress;
import com.google.appengine.api.datastore.Rating;
import com.google.appengine.api.datastore.ShortBlob;
import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.users.User;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
/**
* this utility class is used to convert data among datastore types and DTOs
*
* @author Max Zhu (thebbsky@gmail.com)
*
*/
public class DatastoreUtil {
/**
* @param currEntity
* @return
*/
public static List<Key> withAllAncesterKeys(Key currKey) {
List<Key> keys = new LinkedList<Key>();
if (currKey == null) {
return keys;
}
keys.add(currKey);
while (currKey.getParent() != null) {
keys.add(0, currKey.getParent());
currKey = currKey.getParent();
}
return keys;
}
/**
* @param <T>
* @param iterable
* @return
*/
public static <T> T singleEntityFrom(Iterable<T> iterable) {
Iterator<T> i = iterable.iterator();
return i.hasNext() ? i.next() : null;
}
/**
* @param entities
* @return
*/
public static List<Map<String, Object>> getProperties(Iterable<Entity> entities) {
List<Map<String, Object>> result = new LinkedList<Map<String, Object>>();
for (Entity e : entities) {
result.add(getProperties(e));
}
return result;
}
/**
* @param e
* @return
*/
public static Map<String, Object> getProperties(Entity e) {
if (e == null) {
return new HashMap<String, Object>();
} else {
Builder<String, Object> builder = new ImmutableMap.Builder<String, Object>();
// step 1 : build key hierachy
Key key = e.getKey();
int i = 0;
while (key != null) {
builder.put("key" + (i++), isNullOrEmpty(key.getName()) ? key.getId() : key.getKind() + ":" + key.getName());
key = key.getParent();
}
// step 2 : namespace
builder.put("namespace", e.getNamespace());
// step 3 : all other properties
builder.putAll(e.getProperties());
return builder.build();
}
}
/**
* @param <T>
* @param e
* @param propertyName
* @param clazz
* @return
*/
@SuppressWarnings("unchecked")
public static <T> T getProperty(Entity e, String propertyName, Class<T> clazz) {
if (e == null) {
return null;
}
if (e.hasProperty(propertyName)) {
return (T) e.getProperty(propertyName);
} else {
return null;
}
}
/**
* based on logics documented here:
* {@link http://code.google.com/appengine/docs/python/datastore/entities.html}
*
* @param arg0
* @param arg1
* @return
*/
public static int deterministicCompare(Object arg0, Object arg1) {
int type1 = typeOrder(arg0);
int type2 = typeOrder(arg1);
if (type1 != type2) {
// different type, order by type directly
return type1 - type2;
}
// same type
switch (type1) {
case 0:
// both are null
return 0;
case 1: // long, date or rating
long l1 = longValue(arg0);
long l2 = longValue(arg1);
// use if-else condition to prevent integer overflow
if (l1 == l2) {
return 0;
} else if (l1 < l2) {
return -1;
} else {
return 1;
}
case 2: // boolean
return ((Boolean)arg0).compareTo((Boolean) arg1);
case 3: // shortblob
return ((ShortBlob)arg0).compareTo((ShortBlob) arg1);
case 4: // Unicode strings: text strings (short), category, email address, IM handle, link, telephone number, postal address
String str0 = stringValue(arg0);
String str1 = stringValue(arg1);
return str0.compareTo(str1);
case 5: // Float or Double
BigDecimal bd0 = BigDecimalUtil.of((Number) arg0);
BigDecimal bd1 = BigDecimalUtil.of((Number) arg1);
return bd0.compareTo(bd1);
case 6: // GeoPt
return ((GeoPt)arg0).compareTo((GeoPt) arg1);
case 7: // User
return ((User)arg0).compareTo((User) arg1);
case 8: // Key
return ((Key)arg0).compareTo((Key) arg1);
case 9: // BlobKey
return ((BlobKey)arg0).compareTo((BlobKey) arg1);
default:
//Long text strings and long byte strings are not indexed by the datastore, and so have no ordering defined.
return 0;
}
}
/**
* @param arg0
* @return
*/
private static int typeOrder(Object arg0) {
if (arg0 == null) {
return 0;
} else if (arg0 instanceof Long || arg0 instanceof Date || arg0 instanceof Rating) {
return 1;
} else if (arg0 instanceof Boolean) {
return 2;
} else if (arg0 instanceof ShortBlob) {
return 3;
} else if (arg0 instanceof String || arg0 instanceof Category || arg0 instanceof Email
|| arg0 instanceof IMHandle || arg0 instanceof Link || arg0 instanceof PhoneNumber
|| arg0 instanceof PostalAddress) {
return 4;
} else if (arg0 instanceof Float || arg0 instanceof Double || arg0 instanceof BigDecimal) {
// we put Bigdecimal here because almost all after-process data are in BigDecimal
return 5;
} else if (arg0 instanceof GeoPt) {
return 6;
} else if (arg0 instanceof User) {
return 7;
} else if (arg0 instanceof Key) {
return 8;
} else if (arg0 instanceof BlobKey) {
return 9;
} else {
//Long text strings and long byte strings are not indexed by the datastore, and so have no ordering defined.
return 10;
}
}
/**
* @param arg0
* @return
*/
private static long longValue(Object arg0) {
checkNotNull(arg0);
if (arg0 instanceof Long) {
return (Long) arg0;
} else if (arg0 instanceof Date) {
return ((Date) arg0).getTime();
} else if (arg0 instanceof Rating) {
return ((Rating) arg0).getRating();
} else {
throw new IllegalArgumentException(arg0.getClass() + " is not supported");
}
}
/**
* Unicode strings: text strings (short), category, email address, IM handle, link, telephone number, postal address
*
* @param arg0
* @return
*/
private static String stringValue(Object arg0) {
checkNotNull(arg0);
if (arg0 instanceof String) {
return (String) arg0;
} else if (arg0 instanceof Category) {
return ((Category) arg0).getCategory();
} else if (arg0 instanceof Email) {
return ((Email) arg0).getEmail();
} else if (arg0 instanceof IMHandle) {
return arg0.toString();
} else if (arg0 instanceof Link) {
return ((Link) arg0).getValue();
} else if (arg0 instanceof PhoneNumber) {
return ((PhoneNumber) arg0).getNumber();
} else if (arg0 instanceof PostalAddress) {
return ((PostalAddress) arg0).getAddress();
} else {
throw new IllegalArgumentException(arg0.getClass() + " is not supported");
}
}
private static final String PROPERTY_REPRESENTATION = "property_representation";
/**
* put it here for easy testing
*
* @param kindsMap
* @param propertiesMap
* @return
*/
public static List<MetaNamespace> buildMetaData(Map<String, Iterable<Entity>> kindsMap,
Map<String, Iterable<Entity>> propertiesMap) {
Map<String, MetaNamespace> namespacesMap = new HashMap<String, MetaNamespace>();
// step 1 : populate kinds
for (String namespaceName : kindsMap.keySet()) {
for (Entity kindEntity : kindsMap.get(namespaceName)) {
MetaNamespace namespace = namespacesMap.get(namespaceName);
if (namespace == null) {
namespace = new MetaNamespace(namespaceName);
namespacesMap.put(namespaceName, namespace);
}
String kindName = kindEntity.getKey().getName();
namespace.getKindsMap().put(kindName, new MetaKind(kindName));
}
}
// step 2 : populate properties map
for (String namespaceName : propertiesMap.keySet()) {
for (Entity propertyEntity : propertiesMap.get(namespaceName)) {
Key propertyKey = propertyEntity.getKey();
Key kindKey = propertyKey.getParent();
@SuppressWarnings("unchecked")
List<String> representation =
(List<String>) propertyEntity.getProperty(PROPERTY_REPRESENTATION);
MetaNamespace namespace = namespacesMap.get(namespaceName);
if (namespace == null) {
throw new IllegalArgumentException("unknown namespace " + namespaceName);
}
MetaKind kind = namespace.getKindsMap().get(kindKey.getName());
if (kind == null) {
throw new IllegalArgumentException("unknown kind " + kindKey.getName());
}
kind.addProperty(propertyKey.getName(), representation);
}
}
return new ArrayList<MetaNamespace>(namespacesMap.values());
}
/**
* @param key
* @return
*/
public static KeyInfo convert(Key key) {
if (key == null) {
return null;
}
return new KeyInfo(convert(key.getParent()),
key.getKind(), key.getName(), key.getId(),
KeyFactory.keyToString(key));
}
/**
* convert KeyInfo back to key
*
* @param info
* @return
*/
public static Key convert(KeyInfo info) {
if (info == null) {
return null;
}
Key parent = convert(info.getParent());
if (info.getId() == null || info.getId() == 0l) { // name key
return KeyFactory.createKey(parent, info.getKind(), info.getName());
} else { // id key
return KeyFactory.createKey(parent, info.getKind(), info.getId());
}
}
/**
* @param e
* @return
*/
public static EntityInfo convert(Entity e) {
if (e == null) {
return null;
}
KeyInfo keyInfo = DatastoreUtil.convert(e.getKey());
EntityInfo entityInfo = new EntityInfo(keyInfo);
for (String propertyName : e.getProperties().keySet()) {
entityInfo.getPropertisMap().put(propertyName,
DatastoreUtil.convert(
KeyFactory.keyToString(e.getKey()),
propertyName,
null,
e.getProperty(propertyName),
null)); // warning is always null for direct datastore load
}
return entityInfo;
}
/**
* @param keyString current entity key string
* @param propertyName current property name
* @param index current iterating index (if it's a list)
* @param obj
* @param warnings
* @return
*/
public static PropertyInfo convert(String keyString, String propertyName, Integer index,
Object obj, List<String> warnings) {
PropertyInfo result = null;
if (obj == null) {
result = new NullPropertyInfo();
} else if (obj instanceof Boolean) {
result = new BooleanPropertyInfo((Boolean)obj);
} else if (obj instanceof String) {
result = new StringPropertyInfo((String)obj);
} else if (obj instanceof Category) {
result = new StringPropertyInfo(((Category) obj).getCategory());
} else if (obj instanceof Date) {
result = new DatePropertyInfo((Date)obj);
} else if (obj instanceof Email) {
result = new StringPropertyInfo(((Email) obj).getEmail());
} else if (obj instanceof Long) { // short, int, long are all stored as long value
result = new LongPropertyInfo((Long)obj);
} else if (obj instanceof Double) {// float and double are both stored as 64-bit double precision, IEEE 754
result = new DoublePropertyInfo((Double)obj);
} else if (obj instanceof BigDecimal) { // most processed fields will be bigdecimal
BigDecimal bd = (BigDecimal) obj;
if ((double)bd.longValue() == bd.doubleValue()) { // try to make it long if there is no lose of precision
result = new LongPropertyInfo(bd.longValue());
} else {
result = new DoublePropertyInfo(bd.doubleValue());
}
} else if (obj instanceof User) {
User user = (User)obj;
result = new UserPropertyInfo(user.getAuthDomain(), user.getEmail(),
user.getFederatedIdentity(), user.getUserId(), user.getNickname());
} else if (obj instanceof GeoPt) {
float latitude = ((GeoPt) obj).getLatitude();
float longitude = ((GeoPt) obj).getLongitude();
result = new GeoPtPropertyInfo(latitude, longitude);
} else if (obj instanceof ShortBlob) {
result = new StringPropertyInfo(new String(((ShortBlob) obj).getBytes()));
} else if (obj instanceof Blob) {
Blob b = (Blob) obj;
String fileName = index == null ? propertyName : propertyName + "[" + index + "]";
FileDownloadPath downloadPath = AutoBeanUtil.newFileDownloadPath(
FileDownloadPath.Type.DATASTORE_BLOB, keyString, propertyName, index, fileName, b.getBytes().length);
String pathStr = AutoBeanUtil.encode(FileDownloadPath.class, downloadPath);
result = new BlobPropertyInfo(b.getBytes().length, pathStr);
} else if (obj instanceof BlobStringWrapper) { // user has edited a blob, in string form, it's not in memcache, nor datastore
result = new BlobPropertyInfo(((BlobStringWrapper) obj).getRawString().getBytes());
} else if (obj instanceof BlobFileRefWrapper) { // user has uploaded the blob, but still stay in memcache
FileDownloadPath path = ((BlobFileRefWrapper) obj).getRef();
String pathStr = AutoBeanUtil.encode(FileDownloadPath.class, path);
result = new BlobPropertyInfo(path.getSize(), pathStr);
} else if (obj instanceof BlobKey) {
String blobKeyString = ((BlobKey) obj).getKeyString();
result = new BlobKeyPropertyInfo(blobKeyString);
} else if (obj instanceof Key) {
result = convert((Key)obj);
} else if (obj instanceof Link) {
result = new StringPropertyInfo(((Link) obj).getValue());
} else if (obj instanceof IMHandle) {
String protocol = ((IMHandle) obj).getProtocol();
String address = ((IMHandle) obj).getAddress();
result = new IMHandlePropertyInfo(protocol, address);
} else if (obj instanceof PostalAddress) {
result = new StringPropertyInfo(((PostalAddress) obj).getAddress());
} else if (obj instanceof Rating) {
result = new LongPropertyInfo(((Rating) obj).getRating());
} else if (obj instanceof PhoneNumber) {
result = new StringPropertyInfo(((PhoneNumber) obj).getNumber());
} else if (obj instanceof Text) {
Text t = (Text)obj;
String fileName = index == null ? propertyName : propertyName + "[" + index + "]";
FileDownloadPath downloadPath = AutoBeanUtil.newFileDownloadPath(
FileDownloadPath.Type.DATASTORE_TEXT, keyString, propertyName, index,
fileName, t.getValue().length());
String pathStr = AutoBeanUtil.encode(FileDownloadPath.class, downloadPath);
String fullStr = ((Text) obj).getValue();
result = new TextPropertyInfo(fullStr, pathStr);
} else if (obj instanceof TextStringWrapper) {
result = new TextPropertyInfo(((TextStringWrapper) obj).getRawString());
} else if (obj instanceof TextFileRefWrapper) {
FileDownloadPath path = ((TextFileRefWrapper) obj).getRef();
String pathStr = AutoBeanUtil.encode(FileDownloadPath.class, path);
result = new TextPropertyInfo("No preview available", path.getSize(), pathStr);
} else if (obj instanceof List) {
@SuppressWarnings("rawtypes")
List list = (List) obj;
ListPropertyInfo propertyInfo = new ListPropertyInfo();
int size = list.size();
for (int i = 0 ; i < size ; i ++) {
Object element = list.get(i);
propertyInfo.add(convert(keyString, propertyName, i, element, null));
}
result = propertyInfo;
}
if (result == null) {
throw new IllegalArgumentException("Unexpected type " + obj.getClass().getName());
} else {
result.setTitle(propertyName);
result.setWarnings(warnings);
return result;
}
}
/**
* populate property infos from evaluator and evaluation result
*
* @param evaluator metadata
* @param result data
*/
public static void populatePropertyInfos(Evaluator evaluator, EvaluationResult result, List<PropertyInfo> listToAppend) {
String keyString = result.getKey() == null ?
null : KeyFactory.keyToString(result.getKey());
// try to use property name, as evaluator.getText may not be correct, eg, select * from job
String propertyName = result.getKey() == null ?
evaluator.getText() : result.getPropertyName();
Integer idx = result.getIndex();
Object val = result.getPayload();
if (val instanceof EvaluationResult []) { // select all case, eg: select * from job
for (EvaluationResult r : (EvaluationResult []) val) {
populatePropertyInfos(evaluator, r, listToAppend);
}
} else {
listToAppend.add(convert(keyString, propertyName, idx, val, result.getWarnings()));
}
}
public static Object toDatastoreType(Object obj) {
if (obj instanceof BigDecimal) {
return BigDecimalUtil.toDatastoreNumber((BigDecimal) obj);
} else if (obj instanceof BlobStringWrapper) {
return new Blob(((BlobStringWrapper) obj).getRawString().getBytes());
} else if (obj instanceof TextStringWrapper) {
return new Text(((TextStringWrapper) obj).getRawString());
} else if (obj instanceof BlobFileRefWrapper || obj instanceof TextFileRefWrapper) {
throw new YaacException(null, "File reference can not be used here");
} else {
return obj;
}
}
public static Object ensureEvaluationType(Object obj) {
if (obj instanceof Number) {
return BigDecimalUtil.of((Number) obj);
} else {
return obj;
}
}
}