/*
* Copyright 2012 NGDATA nv
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.lilyproject.tools.import_.json.test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import com.google.common.collect.Lists;
import org.apache.hadoop.hbase.util.Bytes;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.ObjectNode;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.lilyproject.bytes.api.ByteArray;
import org.lilyproject.repository.api.IdGenerator;
import org.lilyproject.repository.api.LRepository;
import org.lilyproject.repository.api.LTable;
import org.lilyproject.repository.api.Metadata;
import org.lilyproject.repository.api.MetadataBuilder;
import org.lilyproject.repository.api.QName;
import org.lilyproject.repository.api.Record;
import org.lilyproject.repository.api.RecordId;
import org.lilyproject.repository.api.RecordScan;
import org.lilyproject.repository.api.RepositoryException;
import org.lilyproject.repository.api.ReturnFields;
import org.lilyproject.repository.api.Scope;
import org.lilyproject.repository.api.TypeManager;
import org.lilyproject.repository.api.filter.FieldValueFilter;
import org.lilyproject.repository.api.filter.RecordFilterList;
import org.lilyproject.repository.api.filter.RecordIdPrefixFilter;
import org.lilyproject.repository.api.filter.RecordTypeFilter;
import org.lilyproject.repository.api.filter.RecordVariantFilter;
import org.lilyproject.repository.impl.id.IdGeneratorImpl;
import org.lilyproject.repotestfw.RepositorySetup;
import org.lilyproject.tools.import_.cli.JsonImport;
import org.lilyproject.tools.import_.json.IgnoreAndDeleteEmptyFieldsRecordReader;
import org.lilyproject.tools.import_.json.JsonFormatException;
import org.lilyproject.tools.import_.json.NamespacesImpl;
import org.lilyproject.tools.import_.json.RecordReader;
import org.lilyproject.tools.import_.json.RecordScanReader;
import org.lilyproject.tools.import_.json.RecordScanWriter;
import org.lilyproject.tools.import_.json.RecordWriter;
import org.lilyproject.tools.import_.json.WriteOptions;
import org.lilyproject.util.io.Closer;
import org.lilyproject.util.json.JsonFormat;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.lilyproject.tools.import_.cli.JsonImport.ImportSettings;
public class JsonConversionTest {
private final static RepositorySetup repoSetup = new RepositorySetup();
private static LRepository repository;
private static LTable table;
private RecordScanWriter writer;
private RecordScanReader reader;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
repoSetup.setupCore();
repoSetup.setupRepository();
repository = repoSetup.getRepositoryManager().getDefaultRepository();
table = repository.getDefaultTable();
TypeManager typeManager = repository.getTypeManager();
typeManager.createFieldType("STRING", new QName("ns", "stringField"), Scope.NON_VERSIONED);
typeManager.createFieldType("LIST<STRING>", new QName("ns", "stringListField"), Scope.NON_VERSIONED);
typeManager.createFieldType("LONG", new QName("ns", "longField"), Scope.NON_VERSIONED);
typeManager.recordTypeBuilder()
.defaultNamespace("ns")
.name("rt")
.fieldEntry()
.name("stringField")
.add()
.fieldEntry()
.name("stringListField")
.add()
.fieldEntry()
.name("longField")
.add()
.create();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
Closer.close(repoSetup);
}
@Before
public void setup() {
writer = new RecordScanWriter();
reader = new RecordScanReader();
}
/**
* Tests that namespaces can be declared globally and/or locally.
*/
@Test
public void testNamespaceContexts() throws Exception {
JsonImport.load(table, repository, getClass().getResourceAsStream("nscontexttest.json"));
Record record1 = table.read(repository.getIdGenerator().fromString("USER.record1"));
assertEquals("value1", record1.getField(new QName("import1", "f1")));
assertEquals(new Integer(55), record1.getField(new QName("import2", "f2")));
}
private byte[] scanToBytes(RecordScan scan) throws RepositoryException, InterruptedException, IOException {
return JsonFormat.serializeAsBytes(
writer.toJson(scan, new WriteOptions(), new NamespacesImpl(false), repository));
}
private RecordScan scanFromBytes(byte[] data) throws IOException, RepositoryException, JsonFormatException,
InterruptedException {
return reader.fromJson(JsonFormat.deserializeNonStd(data), new NamespacesImpl(false), repository);
}
@Test
public void testScanRecordId() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
RecordScan scan = new RecordScan();
scan.setStartRecordId(idGenerator.newRecordId());
scan.setStopRecordId(idGenerator.newRecordId("foo"));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertEquals(scan.getStartRecordId(), parsedScan.getStartRecordId());
assertEquals(scan.getStopRecordId(), parsedScan.getStopRecordId());
// RecordId's should be simply string properties in json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals(scan.getStartRecordId().toString(), node.get("startRecordId").getTextValue());
assertEquals("USER.foo", node.get("stopRecordId").getTextValue());
}
@Test
public void testScanCustomJson() throws Exception {
// verify the json parser accepts comments and unquoted attributes
String json = "{ /* a comment */ startRecordId: \"USER.foo\"}";
scanFromBytes(json.getBytes());
}
@Test
public void testScanRawStartStop() throws Exception {
RecordScan scan = new RecordScan();
scan.setRawStartRecordId(Bytes.toBytes("bar"));
scan.setRawStopRecordId(Bytes.toBytes("foo"));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertArrayEquals(scan.getRawStartRecordId(), parsedScan.getRawStartRecordId());
assertArrayEquals(scan.getRawStopRecordId(), parsedScan.getRawStopRecordId());
// Verify how the bytes are stored in the json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals("YmFy", node.get("rawStartRecordId").getTextValue());
assertEquals("Zm9v", node.get("rawStopRecordId").getTextValue());
}
@Test
public void testScanCaching() throws Exception {
RecordScan scan = new RecordScan();
scan.setCacheBlocks(false);
scan.setCaching(500);
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertEquals(false, parsedScan.getCacheBlocks());
assertEquals(500, parsedScan.getCaching());
}
@Test
public void testScanRecordTypeFilter() throws Exception {
QName recordType = new QName("ns", "rt");
RecordScan scan = new RecordScan();
scan.setRecordFilter(new RecordTypeFilter(recordType));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertNotNull(parsedScan.getRecordFilter());
assertTrue(parsedScan.getRecordFilter() instanceof RecordTypeFilter);
assertEquals(recordType, ((RecordTypeFilter) parsedScan.getRecordFilter()).getRecordType());
assertNull(((RecordTypeFilter) parsedScan.getRecordFilter()).getVersion());
// Check json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals("org.lilyproject.repository.api.filter.RecordTypeFilter",
node.get("recordFilter").get("@class").getTextValue());
assertEquals("{ns}rt",
node.get("recordFilter").get("recordType").getTextValue());
}
@Test
public void testScanRecordIdPrefixFilter() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
RecordId recordId = idGenerator.newRecordId("foo");
RecordScan scan = new RecordScan();
scan.setRecordFilter(new RecordIdPrefixFilter(recordId));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertNotNull(parsedScan.getRecordFilter());
assertTrue(parsedScan.getRecordFilter() instanceof RecordIdPrefixFilter);
assertEquals(recordId, ((RecordIdPrefixFilter) parsedScan.getRecordFilter()).getRecordId());
// Check json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals("org.lilyproject.repository.api.filter.RecordIdPrefixFilter",
node.get("recordFilter").get("@class").getTextValue());
assertEquals("USER.foo",
node.get("recordFilter").get("recordId").getTextValue());
}
@Test
public void testScanFieldValueFilter() throws Exception {
QName name = new QName("ns", "stringField");
Object value = "foo";
RecordScan scan = new RecordScan();
scan.setRecordFilter(new FieldValueFilter(name, value));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertNotNull(parsedScan.getRecordFilter());
assertTrue(parsedScan.getRecordFilter() instanceof FieldValueFilter);
FieldValueFilter filter = (FieldValueFilter) parsedScan.getRecordFilter();
assertEquals(name, filter.getField());
assertEquals(value, filter.getFieldValue());
assertTrue(filter.getFilterIfMissing());
// Check json structure
JsonNode node = new ObjectMapper().readTree(data);
assertEquals("org.lilyproject.repository.api.filter.FieldValueFilter",
node.get("recordFilter").get("@class").getTextValue());
assertEquals("foo", node.get("recordFilter").get("fieldValue").getTextValue());
// Try different data types as field value
value = 3L;
scan.setRecordFilter(new FieldValueFilter(new QName("ns", "longField"), value));
assertEquals(value, ((FieldValueFilter) scanFromBytes(scanToBytes(scan))
.getRecordFilter()).getFieldValue());
value = Lists.newArrayList("foo", "bar");
scan.setRecordFilter(new FieldValueFilter(new QName("ns", "stringListField"), value));
assertEquals(value, ((FieldValueFilter) scanFromBytes(scanToBytes(scan))
.getRecordFilter()).getFieldValue());
// The following test made more sense when we were using a generic type-detection
// in the json serialization rather than using TypeManager.
// Use a list as field value, but with a mixture of datatypes. This should fail,
// as lists in Lily should contain values of the same type.
value = Lists.newArrayList("foo", 123L);
scan.setRecordFilter(new FieldValueFilter(new QName("ns", "stringListField"), value));
try {
data = scanToBytes(scan);
fail("Expected exception with list containing different data types");
} catch (Exception e) {
// expected
}
}
@Test
public void testScanRecordVariantFilter() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
final Map<String, String> variantProperties = new HashMap<String, String>();
variantProperties.put("lang", "en");
variantProperties.put("branch", null);
RecordId recordId = idGenerator.newRecordId("foo");
RecordScan scan = new RecordScan();
scan.setRecordFilter(new RecordVariantFilter(recordId, variantProperties));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertNotNull(parsedScan.getRecordFilter());
assertTrue(parsedScan.getRecordFilter() instanceof RecordVariantFilter);
assertEquals(recordId.getMaster(), ((RecordVariantFilter) parsedScan.getRecordFilter()).getMasterRecordId());
assertEquals(variantProperties, ((RecordVariantFilter) parsedScan.getRecordFilter()).getVariantProperties());
// Check json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals("org.lilyproject.repository.api.filter.RecordVariantFilter",
node.get("recordFilter").get("@class").getTextValue());
assertEquals("USER.foo", node.get("recordFilter").get("recordId").getTextValue());
assertEquals("en", node.get("recordFilter").get("variantProperties").get("lang").getTextValue());
assertEquals(null, node.get("recordFilter").get("variantProperties").get("branch").getTextValue());
}
@Test
public void testScanRecordFilterList() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
RecordId recordId = idGenerator.newRecordId("foo");
RecordScan scan = new RecordScan();
RecordFilterList filterList = new RecordFilterList(RecordFilterList.Operator.MUST_PASS_ONE);
filterList.addFilter(new RecordIdPrefixFilter(recordId));
filterList.addFilter(new RecordTypeFilter(new QName("ns", "stringField")));
scan.setRecordFilter(filterList);
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertNotNull(parsedScan.getRecordFilter());
assertTrue(parsedScan.getRecordFilter() instanceof RecordFilterList);
assertTrue(filterList.getFilters().get(0) instanceof RecordIdPrefixFilter);
assertTrue(filterList.getFilters().get(1) instanceof RecordTypeFilter);
assertEquals(RecordFilterList.Operator.MUST_PASS_ONE, filterList.getOperator());
// Check json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals("org.lilyproject.repository.api.filter.RecordFilterList",
node.get("recordFilter").get("@class").getTextValue());
assertTrue(node.get("recordFilter").get("filters").isArray());
assertEquals(2, node.get("recordFilter").get("filters").size());
assertEquals("org.lilyproject.repository.api.filter.RecordIdPrefixFilter",
node.get("recordFilter").get("filters").get(0).get("@class").getTextValue());
}
@Test
public void testScanReturnFields() throws Exception {
RecordScan scan = new RecordScan();
scan.setReturnFields(ReturnFields.NONE);
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertEquals(ReturnFields.NONE.getType(), parsedScan.getReturnFields().getType());
// Test with enumeration of fields to return
scan.setReturnFields(new ReturnFields(new QName("ns", "f1"), new QName("ns", "f2")));
data = scanToBytes(scan);
parsedScan = scanFromBytes(data);
assertEquals(ReturnFields.Type.ENUM, parsedScan.getReturnFields().getType());
assertEquals(Lists.newArrayList(new QName("ns", "f1"), new QName("ns", "f2")),
parsedScan.getReturnFields().getFields());
}
@Test
public void testScanNamespaces() throws Exception {
QName name = new QName("ns", "stringField");
Object value = "foo";
RecordScan scan = new RecordScan();
scan.setRecordFilter(new FieldValueFilter(name, value));
// Test serialization without namespace prefixes
byte[] data = scanToBytes(scan);
JsonNode node = new ObjectMapper().readTree(data);
assertNull(node.get("namespaces"));
assertEquals("{ns}stringField", node.get("recordFilter").get("field").getTextValue());
// Test serialization with namespace prefixes
byte[] dataWithPrefixes = JsonFormat.serializeAsBytes(
writer.toJson(scan, new WriteOptions(), repository));
JsonNode nodeWithPrefixes = new ObjectMapper().readTree(dataWithPrefixes);
assertNotNull(nodeWithPrefixes.get("namespaces"));
assertTrue(nodeWithPrefixes.get("recordFilter").get("field").getTextValue().endsWith("$stringField"));
}
@Test
public void testRecordAttributes() throws Exception {
Record record = repository.getRecordFactory().newRecord();
record.getAttributes().put("one", "onevalue");
ObjectNode recordNode = RecordWriter.INSTANCE.toJson(record, null, repository);
ObjectNode attributes = (ObjectNode) recordNode.get("attributes");
for (String key : record.getAttributes().keySet()) {
Assert.assertEquals(record.getAttributes().get(key), attributes.get(key).asText());
}
attributes.put("write", "something new");
record = RecordReader.INSTANCE.fromJson(recordNode, repository);
Iterator<Entry<String, JsonNode>> it = attributes.getFields();
while (it.hasNext()) {
Entry<String,JsonNode> attr = it.next();
Assert.assertEquals(attr.getValue().asText(), record.getAttributes().get(attr.getKey()));
}
}
@Test
public void testMetadata() throws Exception {
Record record = repository.getRecordFactory().newRecord();
record.setMetadata(new QName("ns", "field1"),
new MetadataBuilder()
.value("stringfield", "string")
.value("intfield", 55)
.value("longfield", 999999999999L)
.value("booleanfield", Boolean.TRUE)
.value("floatfield", 33.33f)
.value("doublefield", 66.66d)
.value("binaryfield", new ByteArray("foo".getBytes()))
.build());
record.setMetadata(new QName("ns", "field2"),
new MetadataBuilder()
.value("stringfield", "another_string")
.build());
ObjectNode recordNode = RecordWriter.INSTANCE.toJson(record, null, repository);
// go through ser/deser
String recordJson = JsonFormat.serializeAsString(recordNode);
recordNode = (ObjectNode)JsonFormat.deserializeNonStd(recordJson);
// Do some structural validations of the json
ObjectNode metadataNode = (ObjectNode)recordNode.get("metadata");
assertNotNull(metadataNode);
assertNull(recordNode.get("metadataToDelete"));
String prefix = recordNode.get("namespaces").get("ns").getTextValue();
assertEquals(2, metadataNode.size());
assertTrue(metadataNode.get(prefix + "$field1").get("stringfield").isTextual());
assertEquals("string", metadataNode.get(prefix + "$field1").get("stringfield").getTextValue());
assertTrue(metadataNode.get(prefix + "$field1").get("intfield").isInt());
assertEquals(55, metadataNode.get(prefix + "$field1").get("intfield").getIntValue());
assertTrue(metadataNode.get(prefix + "$field1").get("longfield").isLong());
assertEquals(999999999999L, metadataNode.get(prefix + "$field1").get("longfield").getLongValue());
assertTrue(metadataNode.get(prefix + "$field1").get("booleanfield").isBoolean());
assertEquals(Boolean.TRUE, metadataNode.get(prefix + "$field1").get("booleanfield").getBooleanValue());
// in JSON, no distinction between floats & doubles
assertTrue(metadataNode.get(prefix + "$field1").get("floatfield").isFloatingPointNumber());
assertEquals(33.33d, metadataNode.get(prefix + "$field1").get("floatfield").getDoubleValue(), 0.001);
assertTrue(metadataNode.get(prefix + "$field1").get("doublefield").isFloatingPointNumber());
assertEquals(33.33d, metadataNode.get(prefix + "$field1").get("floatfield").getDoubleValue(), 0.001);
assertTrue(metadataNode.get(prefix + "$field1").get("binaryfield").isObject());
ObjectNode binaryNode = (ObjectNode)metadataNode.get(prefix + "$field1").get("binaryfield");
assertEquals("binary", binaryNode.get("type").getTextValue());
assertArrayEquals("foo".getBytes(), binaryNode.get("value").getBinaryValue());
assertTrue(metadataNode.get(prefix + "$field2").get("stringfield").isTextual());
assertEquals("another_string", metadataNode.get(prefix + "$field2").get("stringfield").getTextValue());
// Now parse json again to API objects
record = RecordReader.INSTANCE.fromJson(recordNode, repository);
assertEquals(2, record.getMetadataMap().size());
Metadata metadata = record.getMetadata(new QName("ns", "field1"));
assertEquals("string", metadata.get("stringfield"));
assertEquals(55, metadata.getInt("intfield", null).intValue());
assertEquals(999999999999L, metadata.getLong("longfield", null).longValue());
assertEquals(Boolean.TRUE, metadata.getBoolean("booleanfield", null));
assertEquals(33.33f, metadata.getFloat("floatfield", null), 0.001);
assertEquals(66.66d, metadata.getDouble("doublefield", null), 0.001);
assertEquals(new ByteArray("foo".getBytes()), metadata.getBytes("binaryfield"));
metadata = record.getMetadata(new QName("ns", "field2"));
assertEquals(1, metadata.getMap().size());
assertEquals(0, metadata.getFieldsToDelete().size());
}
@Test
public void testMetadataToDelete() throws Exception {
Record record = repository.getRecordFactory().newRecord();
record.setMetadata(new QName("ns", "field1"),
new MetadataBuilder()
.value("mfield1", "value1")
.delete("mfield2")
.delete("mfield3")
.build());
record.setMetadata(new QName("ns", "field2"),
new MetadataBuilder()
.delete("mfield4")
.build());
WriteOptions options = new WriteOptions();
options.setUseNamespacePrefixes(false);
ObjectNode recordNode = RecordWriter.INSTANCE.toJson(record, options, repository);
// Go through ser/deser
String recordJson = JsonFormat.serializeAsString(recordNode);
recordNode = (ObjectNode)JsonFormat.deserializeNonStd(recordJson);
assertNotNull(recordNode.get("metadataToDelete"));
assertEquals(2, recordNode.get("metadataToDelete").size());
assertEquals(2, recordNode.get("metadataToDelete").get("{ns}field1").size());
assertEquals(1, recordNode.get("metadataToDelete").get("{ns}field2").size());
// Now parse json again to API objects
record = RecordReader.INSTANCE.fromJson(recordNode, repository);
assertEquals(2, record.getMetadataMap().size());
Metadata metadata = record.getMetadata(new QName("ns", "field1"));
assertTrue(metadata.getFieldsToDelete().contains("mfield2"));
assertTrue(metadata.getFieldsToDelete().contains("mfield3"));
assertEquals("value1", metadata.get("mfield1"));
metadata = record.getMetadata(new QName("ns", "field2"));
assertTrue(metadata.getFieldsToDelete().contains("mfield4"));
assertEquals(1, metadata.getFieldsToDelete().size());
}
@Test
public void testLineBasedJsonImport() throws Exception {
JsonImport.loadJsonLines(table, repository,
getClass().getResourceAsStream("json_line_input.txt"), new ImportSettings());
Record record = table.read(repository.getIdGenerator().fromString("USER.jsonline1"));
assertEquals("hello1", record.getField(new QName("ns", "stringField")));
record = table.read(repository.getIdGenerator().fromString("USER.jsonline2"));
assertEquals("hello2", record.getField(new QName("ns", "stringField")));
}
@Test
public void testIgnoreEmptyFields() throws Exception {
ImportSettings settings = new ImportSettings();
settings.recordReader = IgnoreAndDeleteEmptyFieldsRecordReader.INSTANCE;
JsonImport.load(table, repository, getClass().getResourceAsStream("emptyfieldsignore_1.json"), settings);
QName stringField = new QName("ns", "string");
QName integerField = new QName("ns", "integer");
QName stringlist = new QName("ns", "stringlist");
QName nested1 = new QName("ns", "nested1");
QName nested2 = new QName("ns", "nested2");
Record record = table.read(repository.getIdGenerator().fromString("USER.record1"));
assertTrue(record.hasField(stringField));
assertTrue(record.hasField(integerField));
assertTrue(record.hasField(stringlist));
assertTrue(record.hasField(nested1));
assertTrue(record.hasField(nested2));
JsonImport.load(table, repository, getClass().getResourceAsStream("emptyfieldsignore_2.json"), settings);
record = table.read(repository.getIdGenerator().fromString("USER.record1"));
assertFalse(record.hasField(stringField));
assertFalse(record.hasField(integerField));
assertFalse(record.hasField(stringlist));
assertFalse(record.hasField(nested1));
assertTrue(record.hasField(nested2));
Record nestedRecord = record.getField(nested2);
assertFalse(nestedRecord.hasField(new QName("ns", "field1")));
assertTrue(nestedRecord.hasField(new QName("ns", "field2")));
}
}