/*
* Copyright 2010 Outerthought bvba
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.lilyproject.linkindex.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
import com.google.common.collect.Sets;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.lilyproject.hadooptestfw.TestHelper;
import org.lilyproject.hbaseindex.IndexManager;
import org.lilyproject.linkindex.FieldedLink;
import org.lilyproject.linkindex.LinkIndex;
import org.lilyproject.linkindex.LinkIndexUpdater;
import org.lilyproject.repository.api.AbsoluteRecordId;
import org.lilyproject.repository.api.FieldType;
import org.lilyproject.repository.api.HierarchyPath;
import org.lilyproject.repository.api.IdGenerator;
import org.lilyproject.repository.api.Link;
import org.lilyproject.repository.api.QName;
import org.lilyproject.repository.api.Record;
import org.lilyproject.repository.api.RecordId;
import org.lilyproject.repository.api.RecordType;
import org.lilyproject.repository.api.Repository;
import org.lilyproject.repository.api.SchemaId;
import org.lilyproject.repository.api.Scope;
import org.lilyproject.repository.api.TypeManager;
import org.lilyproject.repository.impl.id.SchemaIdImpl;
import org.lilyproject.repotestfw.RepositorySetup;
import org.lilyproject.util.hbase.LilyHBaseSchema.Table;
import org.lilyproject.util.io.Closer;
import org.lilyproject.util.repo.VersionTag;
public class LinkIndexTest {
private static final String TABLE_A = "tableA";
private static final String TABLE_B = "tableB";
private static final String TABLE_C = "tableC";
private final static RepositorySetup repoSetup = new RepositorySetup();
private static TypeManager typeManager;
private static Repository repository;
private static IdGenerator ids;
private static LinkIndex linkIndex;
private SchemaId field1 = new SchemaIdImpl(UUID.randomUUID());
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TestHelper.setupLogging("org.lilyproject.linkindex");
repoSetup.setupCore();
repoSetup.setupRepository();
typeManager = repoSetup.getTypeManager();
repository = (Repository)repoSetup.getRepositoryManager().getDefaultRepository().getDefaultTable();
ids = repository.getIdGenerator();
IndexManager indexManager = new IndexManager(repoSetup.getHadoopConf());
linkIndex = new LinkIndex(indexManager, repoSetup.getRepositoryManager());
repoSetup.getSepModel().addSubscription("LinkIndexUpdater");
repoSetup.getTableManager().createTable(TABLE_A);
repoSetup.getTableManager().createTable(TABLE_B);
repoSetup.getTableManager().createTable(TABLE_C);
repoSetup.startSepEventSlave("LinkIndexUpdater", new LinkIndexUpdater(repoSetup.getRepositoryManager(), linkIndex));
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
Closer.close(repoSetup);
}
@Test
public void testLinkIndex() throws Exception {
SchemaId liveTag = repository.getIdGenerator().getSchemaId(UUID.randomUUID());
Set<FieldedLink> links1 = new HashSet<FieldedLink>();
links1.add(new FieldedLink(createAbsoluteId("id1"), field1));
links1.add(new FieldedLink(createAbsoluteId("id2"), field1));
Set<FieldedLink> links2 = new HashSet<FieldedLink>();
links2.add(new FieldedLink(createAbsoluteId("id3"), field1));
links2.add(new FieldedLink(createAbsoluteId("id4"), field1));
linkIndex.updateLinks(ids.newRecordId("idA"), liveTag, links1);
linkIndex.updateLinks(ids.newRecordId("idB"), liveTag, links1);
linkIndex.updateLinks(ids.newRecordId("idC"), liveTag, links2);
// Test forward link retrieval
Set<FieldedLink> links = linkIndex.getFieldedForwardLinks(ids.newRecordId("idA"), liveTag);
assertTrue(links.contains(new FieldedLink(createAbsoluteId("id1"), field1)));
assertTrue(links.contains(new FieldedLink(createAbsoluteId("id2"), field1)));
assertEquals(2, links.size());
// Test backward link retrieval
Set<RecordId> referrers = linkIndex.getReferrers(ids.newRecordId("id1"), liveTag);
assertTrue(referrers.contains(ids.newRecordId("idA")));
assertTrue(referrers.contains(ids.newRecordId("idB")));
assertEquals(2, referrers.size());
// Update the links for record idA and re-check
links1.add(new FieldedLink(createAbsoluteId("id2a"), field1));
linkIndex.updateLinks(ids.newRecordId("idA"), liveTag, links1);
links = linkIndex.getFieldedForwardLinks(ids.newRecordId("idA"), liveTag);
assertTrue(links.contains(new FieldedLink(createAbsoluteId("id1"), field1)));
assertTrue(links.contains(new FieldedLink(createAbsoluteId("id2"), field1)));
assertTrue(links.contains(new FieldedLink(createAbsoluteId("id2a"), field1)));
assertEquals(3, links.size());
referrers = linkIndex.getReferrers(ids.newRecordId("id1"), liveTag);
assertTrue(referrers.contains(ids.newRecordId("idA")));
assertTrue(referrers.contains(ids.newRecordId("idB")));
assertEquals(2, referrers.size());
referrers = linkIndex.getReferrers(ids.newRecordId("id2a"), liveTag);
assertTrue(referrers.contains(ids.newRecordId("idA")));
assertEquals(1, referrers.size());
}
@Test
public void testLinkIndex_AcrossTables() throws Exception {
SchemaId liveTag = repository.getIdGenerator().getSchemaId(UUID.randomUUID());
Set<FieldedLink> links1 = new HashSet<FieldedLink>();
links1.add(new FieldedLink(ids.newAbsoluteRecordId(TABLE_A, "id1"), field1));
links1.add(new FieldedLink(ids.newAbsoluteRecordId(TABLE_A, "id2"), field1));
Set<FieldedLink> links2 = new HashSet<FieldedLink>();
links2.add(new FieldedLink(ids.newAbsoluteRecordId(TABLE_B, "id3"), field1));
links2.add(new FieldedLink(ids.newAbsoluteRecordId(TABLE_B, "id4"), field1));
linkIndex.updateLinks(ids.newAbsoluteRecordId(TABLE_A, "idA"), liveTag, links1);
linkIndex.updateLinks(ids.newAbsoluteRecordId(TABLE_B, "idB"), liveTag, links1);
linkIndex.updateLinks(ids.newAbsoluteRecordId(TABLE_C, "idC"), liveTag, links2);
// Test forward link retrieval
Set<FieldedLink> links = linkIndex.getFieldedForwardLinks(ids.newAbsoluteRecordId(TABLE_A, "idA"), liveTag);
assertEquals(2, links.size());
assertEquals(
Sets.newHashSet(
new FieldedLink(ids.newAbsoluteRecordId(TABLE_A, "id1"), field1),
new FieldedLink(ids.newAbsoluteRecordId(TABLE_A, "id2"), field1)),
links);
// Test backward link retrieval - non-absolute ids
Set<AbsoluteRecordId> referrers = linkIndex.getAbsoluteReferrers(ids.newAbsoluteRecordId(TABLE_A, "id1"), liveTag);
assertEquals(2, referrers.size());
assertEquals(
Sets.newHashSet(
ids.newAbsoluteRecordId( TABLE_A, "idA"),
ids.newAbsoluteRecordId( TABLE_B, "idB")),
referrers);
// Test backward link retrieval - absolute ids
Set<AbsoluteRecordId> absoluteReferrers = linkIndex.getAbsoluteReferrers(ids.newAbsoluteRecordId(TABLE_A, "id1"), liveTag);
assertTrue(absoluteReferrers.contains(ids.newAbsoluteRecordId(TABLE_A, "idA")));
assertTrue(absoluteReferrers.contains(ids.newAbsoluteRecordId(TABLE_B, "idB")));
assertEquals(2, absoluteReferrers.size());
}
@Test
public void testLinkIndexWithShortRecordIds() throws Exception {
final RecordId id1 = ids.newRecordId("id1");
final RecordId id2 = ids.newRecordId("id2");
final RecordId id3 = ids.newRecordId("id3");
final RecordId id4 = ids.newRecordId("id4");
testLinkIndexRetrievalWithProvidedIds(id1, id2, id3, id4);
}
@Test
public void testLinkIndexWithLongRecordIds() throws Exception {
final RecordId id1 = ids.newRecordId("thisIsARecordIdWhichIsMuchLongerThanTenBytes1");
final RecordId id2 = ids.newRecordId("thisIsARecordIdWhichIsMuchLongerThanTenBytes2");
final RecordId id3 = ids.newRecordId("thisIsARecordIdWhichIsMuchLongerThanTenBytes3");
final RecordId id4 = ids.newRecordId("thisIsARecordIdWhichIsMuchLongerThanTenBytes4");
testLinkIndexRetrievalWithProvidedIds(id1, id2, id3, id4);
}
private void testLinkIndexRetrievalWithProvidedIds(RecordId id1, RecordId id2, RecordId id3, RecordId id4) throws Exception {
SchemaId liveTag = repository.getIdGenerator().getSchemaId(UUID.randomUUID());
Set<FieldedLink> links1 = new HashSet<FieldedLink>();
links1.add(new FieldedLink(createAbsoluteId(id1), field1));
links1.add(new FieldedLink(createAbsoluteId(id2), field1));
Set<FieldedLink> links2 = new HashSet<FieldedLink>();
links2.add(new FieldedLink(createAbsoluteId(id3), field1));
links2.add(new FieldedLink(createAbsoluteId(id4), field1));
linkIndex.updateLinks(ids.newRecordId("idA"), liveTag, links1);
linkIndex.updateLinks(ids.newRecordId("idB"), liveTag, links2);
// Test forward link retrieval
Set<FieldedLink> linksFromA = linkIndex.getFieldedForwardLinks(ids.newRecordId("idA"), liveTag);
assertTrue(linksFromA.contains(new FieldedLink(createAbsoluteId(id1), field1)));
assertTrue(linksFromA.contains(new FieldedLink(createAbsoluteId(id2), field1)));
assertEquals(2, linksFromA.size());
Set<FieldedLink> linksFromB = linkIndex.getFieldedForwardLinks(ids.newRecordId("idB"), liveTag);
assertTrue(linksFromB.contains(new FieldedLink(createAbsoluteId(id3), field1)));
assertTrue(linksFromB.contains(new FieldedLink(createAbsoluteId(id4), field1)));
assertEquals(2, linksFromB.size());
// Test backward link retrieval
Set<RecordId> referrers1 = linkIndex.getReferrers(id1, liveTag);
assertTrue(referrers1.contains(ids.newRecordId("idA")));
assertEquals(1, referrers1.size());
Set<RecordId> referrers2 = linkIndex.getReferrers(id2, liveTag);
assertTrue(referrers2.contains(ids.newRecordId("idA")));
assertEquals(1, referrers2.size());
Set<RecordId> referrers3 = linkIndex.getReferrers(id3, liveTag);
assertTrue(referrers3.contains(ids.newRecordId("idB")));
assertEquals(1, referrers3.size());
Set<RecordId> referrers4 = linkIndex.getReferrers(id4, liveTag);
assertTrue(referrers4.contains(ids.newRecordId("idB")));
assertEquals(1, referrers4.size());
}
@Test
public void testLinkIndexUpdater() throws Exception {
FieldType nonVersionedFt = typeManager.newFieldType(typeManager.getValueType("LINK"),
new QName("ns", "link1"), Scope.NON_VERSIONED);
nonVersionedFt = typeManager.createFieldType(nonVersionedFt);
FieldType versionedFt = typeManager.newFieldType(typeManager.getValueType("LIST<LINK>"),
new QName("ns", "link2"), Scope.VERSIONED);
versionedFt = typeManager.createFieldType(versionedFt);
FieldType versionedMutableFt = typeManager.newFieldType(typeManager.getValueType("LIST<LINK>"),
new QName("ns", "link3"), Scope.VERSIONED_MUTABLE);
versionedMutableFt = typeManager.createFieldType(versionedMutableFt);
FieldType nestedFt = typeManager.newFieldType(typeManager.getValueType("LIST<LIST<PATH<LINK>>>"),
new QName("ns", "nestedLinks"), Scope.NON_VERSIONED);
nestedFt = typeManager.createFieldType(nestedFt);
FieldType complexFt = typeManager.newFieldType(typeManager.getValueType("LIST<RECORD>"),
new QName("ns", "complexLinks"), Scope.NON_VERSIONED);
complexFt = typeManager.createFieldType(complexFt);
RecordType recordType = typeManager.newRecordType(new QName("ns", "MyRecordType"));
recordType.addFieldTypeEntry(typeManager.newFieldTypeEntry(nonVersionedFt.getId(), false));
recordType.addFieldTypeEntry(typeManager.newFieldTypeEntry(versionedFt.getId(), false));
recordType.addFieldTypeEntry(typeManager.newFieldTypeEntry(versionedMutableFt.getId(), false));
recordType.addFieldTypeEntry(typeManager.newFieldTypeEntry(nestedFt.getId(), false));
recordType.addFieldTypeEntry(typeManager.newFieldTypeEntry(complexFt.getId(), false));
recordType = typeManager.createRecordType(recordType);
SchemaId lastVTag = typeManager.getFieldTypeByName(VersionTag.LAST).getId();
//
// Link extraction from a record without versions
//
{
Record record = repository.newRecord();
record.setRecordType(recordType.getName());
record.setField(nonVersionedFt.getName(), new Link(ids.newRecordId("foo1")));
record = repository.create(record);
repoSetup.waitForSepProcessing();
Set<RecordId> referrers = linkIndex.getReferrers(ids.newRecordId("foo1"), lastVTag);
assertEquals(1, referrers.size());
assertTrue(referrers.contains(record.getId()));
referrers = linkIndex.getReferrers(ids.newRecordId("bar1"), lastVTag);
assertEquals(0, referrers.size());
// Now perform an update so that there is a version
record.setField(versionedFt.getName(), Arrays.asList(new Link(ids.newRecordId("foo2")),
new Link(ids.newRecordId("foo3"))));
record = repository.update(record);
repoSetup.waitForSepProcessing();
referrers = linkIndex.getReferrers(ids.newRecordId("foo1"), lastVTag);
assertEquals(1, referrers.size());
assertTrue(referrers.contains(record.getId()));
referrers = linkIndex.getReferrers(ids.newRecordId("foo2"), lastVTag);
assertEquals(1, referrers.size());
assertTrue(referrers.contains(record.getId()));
}
//
// Link extraction from nested types
//
{
Record record = repository
.recordBuilder()
.defaultNamespace("ns")
.recordType("MyRecordType")
.field("nestedLinks",
Arrays.asList(
Arrays.asList(
new HierarchyPath(new Link(ids.newRecordId("nl1"))),
new HierarchyPath(new Link(ids.newRecordId("nl2")))
),
Arrays.asList(
new HierarchyPath(new Link(ids.newRecordId("nl3"))),
new HierarchyPath(new Link(ids.newRecordId("nl4")))
)
))
.create();
repoSetup.waitForSepProcessing();
Set<RecordId> referrers = linkIndex.getReferrers(ids.newRecordId("nl1"), lastVTag);
assertEquals(1, referrers.size());
assertTrue(referrers.contains(record.getId()));
Set<RecordId> forwardLinks = linkIndex.getForwardLinks(record.getId(), lastVTag, nestedFt.getId());
assertEquals(4, forwardLinks.size());
assertTrue(forwardLinks.contains(ids.newRecordId("nl1")));
assertTrue(forwardLinks.contains(ids.newRecordId("nl2")));
assertTrue(forwardLinks.contains(ids.newRecordId("nl3")));
assertTrue(forwardLinks.contains(ids.newRecordId("nl4")));
}
//
// Link extraction from complex types
//
{
Record record = repository
.recordBuilder()
.defaultNamespace("ns")
.recordType("MyRecordType")
.field("complexLinks",
Arrays.asList(
repository
.recordBuilder()
.defaultNamespace("ns")
.recordType("MyRecordType")
.field("link1", new Link(ids.newRecordId("cl1")))
.build(),
repository
.recordBuilder()
.defaultNamespace("ns")
.recordType("MyRecordType")
.field("link1", new Link(ids.newRecordId("cl2")))
.build()
))
.create();
repoSetup.waitForSepProcessing();
Set<RecordId> referrers = linkIndex.getReferrers(ids.newRecordId("cl1"), lastVTag);
assertEquals(1, referrers.size());
assertTrue(referrers.contains(record.getId()));
Set<RecordId> forwardLinks = linkIndex.getForwardLinks(record.getId(), lastVTag, complexFt.getId());
assertEquals(2, forwardLinks.size());
assertTrue(forwardLinks.contains(ids.newRecordId("cl1")));
assertTrue(forwardLinks.contains(ids.newRecordId("cl2")));
}
}
private AbsoluteRecordId createAbsoluteId(String recordIdString) {
return createAbsoluteId(ids.newRecordId(recordIdString));
}
private AbsoluteRecordId createAbsoluteId(RecordId recordId) {
return ids.newAbsoluteRecordId(Table.RECORD.name, recordId);
}
}