/*
* Copyright 2012 NGDATA nv
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.lilyproject.indexer.integration;
import javax.annotation.PreDestroy;
import java.util.Collection;
import java.util.Set;
import com.google.common.collect.Sets;
import org.lilyproject.indexer.model.api.LResultToSolrMapper;
import org.lilyproject.indexer.model.indexerconf.IndexRecordFilter;
import org.lilyproject.indexer.model.util.IndexInfo;
import org.lilyproject.indexer.model.util.IndexesInfo;
import org.lilyproject.plugin.PluginRegistry;
import org.lilyproject.repository.api.FieldType;
import org.lilyproject.repository.api.FieldTypes;
import org.lilyproject.repository.api.IdentityRecordStack;
import org.lilyproject.repository.api.QName;
import org.lilyproject.repository.api.Record;
import org.lilyproject.repository.api.Repository;
import org.lilyproject.repository.api.RepositoryException;
import org.lilyproject.repository.api.SchemaId;
import org.lilyproject.repository.api.TypeManager;
import org.lilyproject.repository.spi.RecordUpdateHook;
import org.lilyproject.util.hbase.RepoAndTableUtil;
import org.lilyproject.util.repo.RecordEvent;
import org.lilyproject.util.repo.RecordEvent.IndexRecordFilterData;
/**
* A record update hook that adds information to the RecordEvent needed to evaluate the
* IndexRecordFilter, especially needed to be able to know what IndexRecordFilter's matched
* on the previous (or deleted) record state. Also allows to make this decision without
* needing to read the complete record.
*/
public class IndexRecordFilterHook implements RecordUpdateHook {
private PluginRegistry pluginRegistry;
private final IndexesInfo indexesInfo;
/**
* Name should be unique among all RecordUpdateHook's. Because the name starts
* with 'org.lilyproject', this hook does not need to be activated through
* configuration.
*/
private final String NAME = "org.lilyproject.IndexRecordFilterHook";
public IndexRecordFilterHook(IndexesInfo indexesInfo) {
this.indexesInfo = indexesInfo;
}
public IndexRecordFilterHook(PluginRegistry pluginRegistry, IndexesInfo indexesInfo) {
this.pluginRegistry = pluginRegistry;
this.indexesInfo = indexesInfo;
pluginRegistry.addPlugin(RecordUpdateHook.class, NAME, this);
}
@PreDestroy
public void destroy() {
// Use same arguments as for addPlugin
pluginRegistry.removePlugin(RecordUpdateHook.class, NAME, this);
}
@Override
public void beforeUpdate(Record record, Record originalRecord, Repository repository, FieldTypes fieldTypes,
RecordEvent recordEvent) throws RepositoryException, InterruptedException {
Collection<IndexInfo> indexInfos = indexesInfo.getIndexInfos();
if (indexInfos.size() > 0) {
TypeManager typeMgr = repository.getTypeManager();
RecordEvent.IndexRecordFilterData idxSel = new RecordEvent.IndexRecordFilterData();
recordEvent.setIndexRecordFilterData(idxSel);
idxSel.setOldRecordExists(true);
idxSel.setNewRecordExists(true);
if (indexesInfo.getRecordFilterDependsOnRecordType()) {
// Because record type names can change, this is not guaranteed to be the same as what gets
// stored in the repo, but that doesn't matter as it is only for indexing purposes.
SchemaId oldRecordTypeId = typeMgr.getRecordTypeByName(originalRecord.getRecordTypeName(), null).getId();
idxSel.setOldRecordType(oldRecordTypeId);
// on update, specifying record type is optional
idxSel.setNewRecordType(record.getRecordTypeName() != null ?
typeMgr.getRecordTypeByName(record.getRecordTypeName(), null).getId() : oldRecordTypeId);
}
Set<QName> names = indexesInfo.getRecordFilterFieldDependencies();
for (QName name : names) {
Object oldValue = null, newValue = null;
if (record.hasField(name)) {
newValue = record.getField(name);
}
if (originalRecord.hasField(name)) {
oldValue = originalRecord.getField(name);
}
if (oldValue != null || newValue != null) {
FieldType type = fieldTypes.getFieldType(name);
addField(type, oldValue, newValue, idxSel);
}
}
calculateIndexInclusion(repository.getRepositoryName(), recordEvent.getTableName(), originalRecord, record,
idxSel);
}
}
@Override
public void beforeCreate(Record newRecord, Repository repository, FieldTypes fieldTypes, RecordEvent recordEvent)
throws RepositoryException, InterruptedException {
Collection<IndexInfo> indexInfos = indexesInfo.getIndexInfos();
if (indexInfos.size() > 0) {
TypeManager typeMgr = repository.getTypeManager();
RecordEvent.IndexRecordFilterData idxSel = new RecordEvent.IndexRecordFilterData();
recordEvent.setIndexRecordFilterData(idxSel);
idxSel.setOldRecordExists(false);
idxSel.setNewRecordExists(true);
if (indexesInfo.getRecordFilterDependsOnRecordType()) {
idxSel.setNewRecordType(typeMgr.getRecordTypeByName(newRecord.getRecordTypeName(), null).getId());
}
Set<QName> names = indexesInfo.getRecordFilterFieldDependencies();
for (QName name : names) {
if (newRecord.hasField(name)) {
Object newValue = newRecord.getField(name);
FieldType type = fieldTypes.getFieldType(name);
addField(type, null, newValue, idxSel);
}
}
calculateIndexInclusion(repository.getRepositoryName(), recordEvent.getTableName(), null, newRecord,
idxSel);
}
}
@Override
public void beforeDelete(Record originalRecord, Repository repository, FieldTypes fieldTypes,
RecordEvent recordEvent) throws RepositoryException, InterruptedException {
Collection<IndexInfo> indexInfos = indexesInfo.getIndexInfos();
if (indexInfos.size() > 0) {
TypeManager typeMgr = repository.getTypeManager();
RecordEvent.IndexRecordFilterData idxSel = new RecordEvent.IndexRecordFilterData();
recordEvent.setIndexRecordFilterData(idxSel);
idxSel.setOldRecordExists(true);
idxSel.setNewRecordExists(false);
if (indexesInfo.getRecordFilterDependsOnRecordType()) {
idxSel.setOldRecordType(typeMgr.getRecordTypeByName(originalRecord.getRecordTypeName(), null).getId());
}
Set<QName> names = indexesInfo.getRecordFilterFieldDependencies();
for (QName name : names) {
if (originalRecord.hasField(name)) {
Object oldValue = originalRecord.getField(name);
FieldType type = fieldTypes.getFieldType(name);
addField(type, oldValue, null, idxSel);
}
}
calculateIndexInclusion(repository.getRepositoryName(), recordEvent.getTableName(), originalRecord, null,
idxSel);
}
}
private void addField(FieldType type, Object oldValue, Object newValue, RecordEvent.IndexRecordFilterData idxSel)
throws RepositoryException, InterruptedException {
if (oldValue == null && newValue == null) {
return;
}
if (oldValue != null) {
oldValue = type.getValueType().toBytes(oldValue, new IdentityRecordStack());
}
if (newValue != null) {
newValue = type.getValueType().toBytes(newValue, new IdentityRecordStack());
}
idxSel.addChangedField(type.getId(), (byte[])oldValue, (byte[])newValue);
}
/**
* Calculate the inclusion/exclusion sets for index subscriptions based on the old and new
* records, and update the {@code IndexRecordFilterData} with this information.
*
* @param table table where the record update has occurred
* @param oldRecord Previous version of the record, null if the record is being newly created
* @param newRecord New version of the record, null if the record is being deleted
* @param indexFilterData To be updated with index subscription inclusion/exclusion information
*/
void calculateIndexInclusion(String repositoryName, String table,
Record oldRecord, Record newRecord, IndexRecordFilterData indexFilterData) {
Set<String> applicableIndexes = Sets.newHashSet();
Set<String> nonApplicableIndexes = Sets.newHashSet();
for (IndexInfo indexInfo : indexesInfo.getIndexInfos()) {
String queueSubscriptionId = indexInfo.getIndexDefinition().getSubscriptionId();
if (indexMatchesRepository(repositoryName, indexInfo) &&
indexIsApplicable(indexInfo.getLilyIndexerConf().getRecordFilter(), table, oldRecord, newRecord)) {
applicableIndexes.add(queueSubscriptionId);
} else {
nonApplicableIndexes.add(queueSubscriptionId);
}
}
if (applicableIndexes.isEmpty()) {
indexFilterData.setSubscriptionExclusions(IndexRecordFilterData.ALL_INDEX_SUBSCRIPTIONS);
} else if (nonApplicableIndexes.isEmpty()) {
indexFilterData.setSubscriptionInclusions(IndexRecordFilterData.ALL_INDEX_SUBSCRIPTIONS);
} else if (applicableIndexes.size() > nonApplicableIndexes.size()) {
indexFilterData.setSubscriptionInclusions(applicableIndexes);
} else {
indexFilterData.setSubscriptionExclusions(nonApplicableIndexes);
}
}
boolean indexMatchesRepository(String repositoryName, IndexInfo indexInfo){
String indexRepo = indexInfo.getRepositoryName();
indexRepo = (indexRepo != null ? indexRepo : RepoAndTableUtil.DEFAULT_REPOSITORY);
return indexRepo.equals(repositoryName);
}
/**
* Determine if an {@code IndexRecordFilter} is applicable for either the old or new version of
* a record.
*
* @return true if the index is applicable for either the new or old version of the record
*/
boolean indexIsApplicable(IndexRecordFilter filter, String table, Record oldRecord, Record newRecord) {
return ((oldRecord != null && filter.getIndexCase(table, oldRecord) != null)
|| (newRecord != null && filter.getIndexCase(table, newRecord) != null));
}
}