// This file is part of OpenTSDB.
// Copyright (C) 2013 The OpenTSDB Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 2.1 of the License, or (at your
// option) any later version. This program is distributed in the hope that it
// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
// General Public License for more details. You should have received a copy
// of the GNU Lesser General Public License along with this program. If not,
// see <http://www.gnu.org/licenses/>.
package net.opentsdb.tree;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import net.opentsdb.core.TSDB;
import net.opentsdb.uid.UniqueId;
import net.opentsdb.utils.JSON;
import net.opentsdb.utils.JSONException;
import org.hbase.async.Bytes;
import org.hbase.async.DeleteRequest;
import org.hbase.async.GetRequest;
import org.hbase.async.HBaseException;
import org.hbase.async.KeyValue;
import org.hbase.async.PutRequest;
import org.hbase.async.Scanner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.core.JsonGenerator;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
/**
* Represents a meta data tree in OpenTSDB that organizes timeseries into a
* hierarchical structure for navigation similar to a file system directory.
* Actual results are stored in {@link Branch} and {@link Leaf} objects while
* meta data about the tree is contained in this object.
* <p>
* A tree is built from a set of {@link TreeRule}s. The rules are stored
* separately in the same row as the tree definition object, but can be loaded
* into the tree for processing and return from an RPC request. Building a tree
* consists of defining a tree, assigning one or more rules, and passing
* {@link net.opentsdb.meta.TSMeta} objects through the rule set using a
* {@link TreeBuilder}. Results are then stored in separate rows as branch
* and leaf objects.
* <p>
* If TSMeta collides with something that has already been processed by a
* rule set, a collision will be recorded, via this object, in a separate column
* in a separate row for collisions. Likewise, if a tree is set to
* {@code strict_match}, TSMetas that fail to match the rule set will be
* recorded to a separate row. This class provides helper methods for fetching
* and storing these collisions and non-matched items.
* @since 2.0
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY)
public final class Tree {
private static final Logger LOG = LoggerFactory.getLogger(Tree.class);
/** Charset used to convert Strings to byte arrays and back. */
private static final Charset CHARSET = Charset.forName("ISO-8859-1");
/** Width of tree IDs in bytes */
private static final short TREE_ID_WIDTH = 2;
/** Name of the CF where trees and branches are stored */
private static final byte[] TREE_FAMILY = "t".getBytes(CHARSET);
/** The tree qualifier */
private static final byte[] TREE_QUALIFIER = "tree".getBytes(CHARSET);
/** Integer width in bytes */
private static final short INT_WIDTH = 4;
/** Byte suffix for collision rows, appended after the tree ID */
private static byte COLLISION_ROW_SUFFIX = 0x01;
/** Byte prefix for collision columns */
private static byte[] COLLISION_PREFIX = "tree_collision:".getBytes(CHARSET);
/** Byte suffix for not matched rows, appended after the tree ID */
private static byte NOT_MATCHED_ROW_SUFFIX = 0x02;
/** Byte prefix for not matched columns */
private static byte[] NOT_MATCHED_PREFIX = "tree_not_matched:".getBytes(CHARSET);
/** The numeric ID of this tree object */
private int tree_id;
/** Name of the tree */
private String name = "";
/** A brief description of the tree */
private String description = "";
/** Notes about the tree */
private String notes = "";
/** Whether or not strict matching is enabled */
private boolean strict_match;
/** Whether or not the tree should process meta data or not */
private boolean enabled;
/** Whether or not to store not matched and collisions */
private boolean store_failures;
/** Sorted, two dimensional map of the tree's rules */
private TreeMap<Integer, TreeMap<Integer, TreeRule>> rules;
/** List of non-matched TSUIDs that were not included in the tree */
private HashMap<String, String> not_matched;
/** List of TSUID collisions that were not included in the tree */
private HashMap<String, String> collisions;
/** Unix time, in seconds, when the tree was created */
private long created;
/** Tracks fields that have changed by the user to avoid overwrites */
private final HashMap<String, Boolean> changed =
new HashMap<String, Boolean>();
/**
* Default constructor necessary for de/serialization
*/
public Tree() {
initializeChangedMap();
}
/**
* Constructor that sets the tree ID and the created timestamp to the current
* time.
* @param tree_id ID of this tree
*/
public Tree(final int tree_id) {
this.tree_id = tree_id;
this.created = System.currentTimeMillis() / 1000;
initializeChangedMap();
}
/**
* Copy constructor that creates a completely independent copy of the original
* object.
* @param original The original object to copy from
* @throws PatternSyntaxException if one of the rule's regex is invalid
*/
public Tree(final Tree original) {
created = original.created;
description = original.description;
enabled = original.enabled;
store_failures = original.store_failures;
name = original.name;
notes = original.notes;
strict_match = original.strict_match;
tree_id = original.tree_id;
// deep copy rules
rules = new TreeMap<Integer, TreeMap<Integer, TreeRule>>();
for (Map.Entry<Integer, TreeMap<Integer, TreeRule>> level :
original.rules.entrySet()) {
final TreeMap<Integer, TreeRule> orders = new TreeMap<Integer, TreeRule>();
for (final TreeRule rule : level.getValue().values()) {
orders.put(rule.getOrder(), new TreeRule(rule));
}
rules.put(level.getKey(), orders);
}
// copy collisions and not matched
if (original.collisions != null) {
collisions = new HashMap<String, String>(original.collisions);
}
if (original.not_matched != null) {
not_matched = new HashMap<String, String>(original.not_matched);
}
}
/** @return Information about the tree */
@Override
public String toString() {
return "treeId: " + tree_id + " name: " + name;
}
/**
* Copies changes from the incoming tree into the local tree, overriding if
* called to. Only parses user mutable fields, excluding rules.
* @param tree The tree to copy from
* @param overwrite Whether or not to copy all values from the incoming tree
* @return True if there were changes, false if not
* @throws IllegalArgumentException if the incoming tree was invalid
*/
public boolean copyChanges(final Tree tree, final boolean overwrite) {
if (tree == null) {
throw new IllegalArgumentException("Cannot copy a null tree");
}
if (tree_id != tree.tree_id) {
throw new IllegalArgumentException("Tree IDs do not match");
}
if (overwrite || tree.changed.get("name")) {
name = tree.name;
changed.put("name", true);
}
if (overwrite || tree.changed.get("description")) {
description = tree.description;
changed.put("description", true);
}
if (overwrite || tree.changed.get("notes")) {
notes = tree.notes;
changed.put("notes", true);
}
if (overwrite || tree.changed.get("strict_match")) {
strict_match = tree.strict_match;
changed.put("strict_match", true);
}
if (overwrite || tree.changed.get("enabled")) {
enabled = tree.enabled;
changed.put("enabled", true);
}
if (overwrite || tree.changed.get("store_failures")) {
store_failures = tree.store_failures;
changed.put("store_failures", true);
}
for (boolean has_changes : changed.values()) {
if (has_changes) {
return true;
}
}
return false;
}
/**
* Adds the given rule to the tree, replacing anything in the designated spot
* @param rule The rule to add
* @throws IllegalArgumentException if the incoming rule was invalid
*/
public void addRule(final TreeRule rule) {
if (rule == null) {
throw new IllegalArgumentException("Null rules are not accepted");
}
if (rules == null) {
rules = new TreeMap<Integer, TreeMap<Integer, TreeRule>>();
}
TreeMap<Integer, TreeRule> level = rules.get(rule.getLevel());
if (level == null) {
level = new TreeMap<Integer, TreeRule>();
level.put(rule.getOrder(), rule);
rules.put(rule.getLevel(), level);
} else {
level.put(rule.getOrder(), rule);
}
changed.put("rules", true);
}
/**
* Adds a TSUID to the collision local list, must then be synced with storage
* @param tsuid TSUID to add to the set
* @throws IllegalArgumentException if the tsuid was invalid
*/
public void addCollision(final String tsuid, final String existing_tsuid) {
if (tsuid == null || tsuid.isEmpty()) {
throw new IllegalArgumentException("Empty or null collisions not allowed");
}
if (collisions == null) {
collisions = new HashMap<String, String>();
}
if (!collisions.containsKey(tsuid)) {
collisions.put(tsuid, existing_tsuid);
changed.put("collisions", true);
}
}
/**
* Adds a TSUID to the not-matched local list when strict_matching is enabled.
* Must be synced with storage.
* @param tsuid TSUID to add to the set
* @throws IllegalArgumentException if the tsuid was invalid
*/
public void addNotMatched(final String tsuid, final String message) {
if (tsuid == null || tsuid.isEmpty()) {
throw new IllegalArgumentException("Empty or null non matches not allowed");
}
if (not_matched == null) {
not_matched = new HashMap<String, String>();
}
if (!not_matched.containsKey(tsuid)) {
not_matched.put(tsuid, message);
changed.put("not_matched", true);
}
}
/**
* Attempts to store the tree definition via a CompareAndSet call.
* @param tsdb The TSDB to use for access
* @param overwrite Whether or not tree data should be overwritten
* @return True if the write was successful, false if an error occurred
* @throws IllegalArgumentException if the tree ID is missing or invalid
* @throws HBaseException if a storage exception occurred
*/
public Deferred<Boolean> storeTree(final TSDB tsdb, final boolean overwrite) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Invalid Tree ID");
}
// if there aren't any changes, save time and bandwidth by not writing to
// storage
boolean has_changes = false;
for (Map.Entry<String, Boolean> entry : changed.entrySet()) {
if (entry.getValue()) {
has_changes = true;
break;
}
}
if (!has_changes) {
LOG.debug(this + " does not have changes, skipping sync to storage");
throw new IllegalStateException("No changes detected in the tree");
}
/**
* Callback executed after loading a tree from storage so that we can
* synchronize changes to the meta data and write them back to storage.
*/
final class StoreTreeCB implements Callback<Deferred<Boolean>, Tree> {
final private Tree local_tree;
public StoreTreeCB(final Tree local_tree) {
this.local_tree = local_tree;
}
/**
* Synchronizes the stored tree object (if found) with the local tree
* and issues a CAS call to write the update to storage.
* @return True if the CAS was successful, false if something changed
* in flight
*/
@Override
public Deferred<Boolean> call(final Tree fetched_tree) throws Exception {
Tree stored_tree = fetched_tree;
final byte[] original_tree = stored_tree == null ? new byte[0] :
stored_tree.toStorageJson();
// now copy changes
if (stored_tree == null) {
stored_tree = local_tree;
} else {
stored_tree.copyChanges(local_tree, overwrite);
}
// reset the change map so we don't keep writing
initializeChangedMap();
final PutRequest put = new PutRequest(tsdb.treeTable(),
Tree.idToBytes(tree_id), TREE_FAMILY, TREE_QUALIFIER,
stored_tree.toStorageJson());
return tsdb.getClient().compareAndSet(put, original_tree);
}
}
// initiate the sync by attempting to fetch an existing tree from storage
return fetchTree(tsdb, tree_id).addCallbackDeferring(new StoreTreeCB(this));
}
/**
* Retrieves a single rule from the rule set given a level and order
* @param level The level where the rule resides
* @param order The order in the level where the rule resides
* @return The rule if found, null if not found
*/
public TreeRule getRule(final int level, final int order) {
if (rules == null || rules.isEmpty()) {
return null;
}
TreeMap<Integer, TreeRule> rule_level = rules.get(level);
if (rule_level == null || rule_level.isEmpty()) {
return null;
}
return rule_level.get(order);
}
/**
* Attempts to store the local tree in a new row, automatically assigning a
* new tree ID and returning the value.
* This method will scan the UID table for the maximum tree ID, increment it,
* store the new tree, and return the new ID. If no trees have been created,
* the returned ID will be "1". If we have reached the limit of trees for the
* system, as determined by {@link #TREE_ID_WIDTH}, we will throw an exception.
* @param tsdb The TSDB to use for storage access
* @return A positive ID, greater than 0 if successful, 0 if there was
* an error
*/
public Deferred<Integer> createNewTree(final TSDB tsdb) {
if (tree_id > 0) {
throw new IllegalArgumentException("Tree ID has already been set");
}
if (name == null || name.isEmpty()) {
throw new IllegalArgumentException("Tree was missing the name");
}
/**
* Called after a successful CAS to store the new tree with the new ID.
* Returns the new ID if successful, 0 if there was an error
*/
final class CreatedCB implements Callback<Deferred<Integer>, Boolean> {
@Override
public Deferred<Integer> call(final Boolean cas_success)
throws Exception {
return Deferred.fromResult(tree_id);
}
}
/**
* Called after fetching all trees. Loops through the tree definitions and
* determines the max ID so we can increment and write a new one
*/
final class CreateNewCB implements Callback<Deferred<Integer>, List<Tree>> {
@Override
public Deferred<Integer> call(List<Tree> trees) throws Exception {
int max_id = 0;
if (trees != null) {
for (Tree tree : trees) {
if (tree.tree_id > max_id) {
max_id = tree.tree_id;
}
}
}
tree_id = max_id + 1;
if (tree_id > 65535) {
throw new IllegalStateException("Exhausted all Tree IDs");
}
return storeTree(tsdb, true).addCallbackDeferring(new CreatedCB());
}
}
// starts the process by fetching all tree definitions from storage
return fetchAllTrees(tsdb).addCallbackDeferring(new CreateNewCB());
}
/**
* Attempts to fetch the given tree from storage, loading the rule set at
* the same time.
* @param tsdb The TSDB to use for access
* @param tree_id The Tree to fetch
* @return A tree object if found, null if the tree did not exist
* @throws IllegalArgumentException if the tree ID was invalid
* @throws HBaseException if a storage exception occurred
* @throws JSONException if the object could not be deserialized
*/
public static Deferred<Tree> fetchTree(final TSDB tsdb, final int tree_id) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Invalid Tree ID");
}
// fetch the whole row
final GetRequest get = new GetRequest(tsdb.treeTable(), idToBytes(tree_id));
get.family(TREE_FAMILY);
/**
* Called from the GetRequest with results from storage. Loops through the
* columns and loads the tree definition and rules
*/
final class FetchTreeCB implements Callback<Deferred<Tree>,
ArrayList<KeyValue>> {
@Override
public Deferred<Tree> call(ArrayList<KeyValue> row) throws Exception {
if (row == null || row.isEmpty()) {
return Deferred.fromResult(null);
}
final Tree tree = new Tree();
// WARNING: Since the JSON in storage doesn't store the tree ID, we need
// to loadi t from the row key.
tree.setTreeId(bytesToId(row.get(0).key()));
for (KeyValue column : row) {
if (Bytes.memcmp(TREE_QUALIFIER, column.qualifier()) == 0) {
// it's *this* tree. We deserialize to a new object and copy
// since the columns could be in any order and we may get a rule
// before the tree object
final Tree local_tree = JSON.parseToObject(column.value(), Tree.class);
tree.created = local_tree.created;
tree.description = local_tree.description;
tree.name = local_tree.name;
tree.notes = local_tree.notes;
tree.strict_match = local_tree.strict_match;
tree.enabled = local_tree.enabled;
tree.store_failures = local_tree.store_failures;
// Tree rule
} else if (Bytes.memcmp(TreeRule.RULE_PREFIX(), column.qualifier(), 0,
TreeRule.RULE_PREFIX().length) == 0) {
final TreeRule rule = TreeRule.parseFromStorage(column);
tree.addRule(rule);
}
}
return Deferred.fromResult(tree);
}
}
// issue the get request
return tsdb.getClient().get(get).addCallbackDeferring(new FetchTreeCB());
}
/**
* Attempts to retrieve all trees from the UID table, including their rules.
* If no trees were found, the result will be an empty list
* @param tsdb The TSDB to use for storage
* @return A list of tree objects. May be empty if none were found
*/
public static Deferred<List<Tree>> fetchAllTrees(final TSDB tsdb) {
final Deferred<List<Tree>> result = new Deferred<List<Tree>>();
/**
* Scanner callback that recursively calls itself to load the next set of
* rows from storage. When the scanner returns a null, the callback will
* return with the list of trees discovered.
*/
final class AllTreeScanner implements Callback<Object,
ArrayList<ArrayList<KeyValue>>> {
private final List<Tree> trees = new ArrayList<Tree>();
private final Scanner scanner;
public AllTreeScanner() {
scanner = setupAllTreeScanner(tsdb);
}
/**
* Fetches the next set of results from the scanner and adds this class
* as a callback.
* @return A list of trees if the scanner has reached the end
*/
public Object fetchTrees() {
return scanner.nextRows().addCallback(this);
}
@Override
public Object call(ArrayList<ArrayList<KeyValue>> rows)
throws Exception {
if (rows == null) {
result.callback(trees);
return null;
}
for (ArrayList<KeyValue> row : rows) {
final Tree tree = new Tree();
for (KeyValue column : row) {
if (column.qualifier().length >= TREE_QUALIFIER.length &&
Bytes.memcmp(TREE_QUALIFIER, column.qualifier()) == 0) {
// it's *this* tree. We deserialize to a new object and copy
// since the columns could be in any order and we may get a rule
// before the tree object
final Tree local_tree = JSON.parseToObject(column.value(),
Tree.class);
tree.created = local_tree.created;
tree.description = local_tree.description;
tree.name = local_tree.name;
tree.notes = local_tree.notes;
tree.strict_match = local_tree.strict_match;
tree.enabled = local_tree.enabled;
tree.store_failures = local_tree.store_failures;
// WARNING: Since the JSON data in storage doesn't contain the tree
// ID, we need to parse it from the row key
tree.setTreeId(bytesToId(row.get(0).key()));
// tree rule
} else if (column.qualifier().length > TreeRule.RULE_PREFIX().length &&
Bytes.memcmp(TreeRule.RULE_PREFIX(), column.qualifier(),
0, TreeRule.RULE_PREFIX().length) == 0) {
final TreeRule rule = TreeRule.parseFromStorage(column);
tree.addRule(rule);
}
}
// only add the tree if we parsed a valid ID
if (tree.tree_id > 0) {
trees.add(tree);
}
}
// recurse to get the next set of rows from the scanner
return fetchTrees();
}
}
// start the scanning process
new AllTreeScanner().fetchTrees();
return result;
}
/**
* Returns the collision set from storage for the given tree, optionally for
* only the list of TSUIDs provided.
* <b>Note:</b> This can potentially be a large list if the rule set was
* written poorly and there were many timeseries so only call this
* without a list of TSUIDs if you feel confident the number is small.
* @param tsdb TSDB to use for storage access
* @param tree_id ID of the tree to fetch collisions for
* @param tsuids An optional list of TSUIDs to fetch collisions for. This may
* be empty or null, in which case all collisions for the tree will be
* returned.
* @return A list of collisions or null if nothing was found
* @throws HBaseException if there was an issue
* @throws IllegalArgumentException if the tree ID was invalid
*/
public static Deferred<Map<String, String>> fetchCollisions(final TSDB tsdb,
final int tree_id, final List<String> tsuids) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Invalid Tree ID");
}
final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
row_key[TREE_ID_WIDTH] = COLLISION_ROW_SUFFIX;
final GetRequest get = new GetRequest(tsdb.treeTable(), row_key);
get.family(TREE_FAMILY);
// if the caller provided a list of TSUIDs, then we need to compile a list
// of qualifiers so we only fetch those columns.
if (tsuids != null && !tsuids.isEmpty()) {
final byte[][] qualifiers = new byte[tsuids.size()][];
int index = 0;
for (String tsuid : tsuids) {
final byte[] qualifier = new byte[COLLISION_PREFIX.length +
(tsuid.length() / 2)];
System.arraycopy(COLLISION_PREFIX, 0, qualifier, 0,
COLLISION_PREFIX.length);
final byte[] tsuid_bytes = UniqueId.stringToUid(tsuid);
System.arraycopy(tsuid_bytes, 0, qualifier, COLLISION_PREFIX.length,
tsuid_bytes.length);
qualifiers[index] = qualifier;
index++;
}
get.qualifiers(qualifiers);
}
/**
* Called after issuing the row get request to parse out the results and
* compile the list of collisions.
*/
final class GetCB implements Callback<Deferred<Map<String, String>>,
ArrayList<KeyValue>> {
@Override
public Deferred<Map<String, String>> call(final ArrayList<KeyValue> row)
throws Exception {
if (row == null || row.isEmpty()) {
final Map<String, String> empty = new HashMap<String, String>(0);
return Deferred.fromResult(empty);
}
final Map<String, String> collisions =
new HashMap<String, String>(row.size());
for (KeyValue column : row) {
if (column.qualifier().length > COLLISION_PREFIX.length &&
Bytes.memcmp(COLLISION_PREFIX, column.qualifier(), 0,
COLLISION_PREFIX.length) == 0) {
final byte[] parsed_tsuid = Arrays.copyOfRange(column.qualifier(),
COLLISION_PREFIX.length, column.qualifier().length);
collisions.put(UniqueId.uidToString(parsed_tsuid),
new String(column.value(), CHARSET));
}
}
return Deferred.fromResult(collisions);
}
}
return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}
/**
* Returns the not-matched set from storage for the given tree, optionally for
* only the list of TSUIDs provided.
* <b>Note:</b> This can potentially be a large list if the rule set was
* written poorly and there were many timeseries so only call this
* without a list of TSUIDs if you feel confident the number is small.
* @param tsdb TSDB to use for storage access
* @param tree_id ID of the tree to fetch non matches for
* @param tsuids An optional list of TSUIDs to fetch non-matches for. This may
* be empty or null, in which case all non-matches for the tree will be
* returned.
* @return A list of not-matched mappings or null if nothing was found
* @throws HBaseException if there was an issue
* @throws IllegalArgumentException if the tree ID was invalid
*/
public static Deferred<Map<String, String>> fetchNotMatched(final TSDB tsdb,
final int tree_id, final List<String> tsuids) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Invalid Tree ID");
}
final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
row_key[TREE_ID_WIDTH] = NOT_MATCHED_ROW_SUFFIX;
final GetRequest get = new GetRequest(tsdb.treeTable(), row_key);
get.family(TREE_FAMILY);
// if the caller provided a list of TSUIDs, then we need to compile a list
// of qualifiers so we only fetch those columns.
if (tsuids != null && !tsuids.isEmpty()) {
final byte[][] qualifiers = new byte[tsuids.size()][];
int index = 0;
for (String tsuid : tsuids) {
final byte[] qualifier = new byte[NOT_MATCHED_PREFIX.length +
(tsuid.length() / 2)];
System.arraycopy(NOT_MATCHED_PREFIX, 0, qualifier, 0,
NOT_MATCHED_PREFIX.length);
final byte[] tsuid_bytes = UniqueId.stringToUid(tsuid);
System.arraycopy(tsuid_bytes, 0, qualifier, NOT_MATCHED_PREFIX.length,
tsuid_bytes.length);
qualifiers[index] = qualifier;
index++;
}
get.qualifiers(qualifiers);
}
/**
* Called after issuing the row get request to parse out the results and
* compile the list of collisions.
*/
final class GetCB implements Callback<Deferred<Map<String, String>>,
ArrayList<KeyValue>> {
@Override
public Deferred<Map<String, String>> call(final ArrayList<KeyValue> row)
throws Exception {
if (row == null || row.isEmpty()) {
final Map<String, String> empty = new HashMap<String, String>(0);
return Deferred.fromResult(empty);
}
Map<String, String> not_matched = new HashMap<String, String>(row.size());
for (KeyValue column : row) {
final byte[] parsed_tsuid = Arrays.copyOfRange(column.qualifier(),
NOT_MATCHED_PREFIX.length, column.qualifier().length);
not_matched.put(UniqueId.uidToString(parsed_tsuid),
new String(column.value(), CHARSET));
}
return Deferred.fromResult(not_matched);
}
}
return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}
/**
* Attempts to delete all branches, leaves, collisions and not-matched entries
* for the given tree. Optionally can delete the tree definition and rules as
* well.
* <b>Warning:</b> This call can take a long time to complete so it should
* only be done from a command line or issues once via RPC and allowed to
* process. Multiple deletes running at the same time on the same tree
* shouldn't be an issue but it's a waste of resources.
* @param tsdb The TSDB to use for storage access
* @param tree_id ID of the tree to delete
* @param delete_definition Whether or not the tree definition and rule set
* should be deleted as well
* @return True if the deletion completed successfully, false if there was an
* issue.
* @throws HBaseException if there was an issue
* @throws IllegalArgumentException if the tree ID was invalid
*/
public static Deferred<Boolean> deleteTree(final TSDB tsdb,
final int tree_id, final boolean delete_definition) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Invalid Tree ID");
}
// scan all of the rows starting with the tree ID. We can't just delete the
// rows as there may be other types of data. Thus we have to check the
// qualifiers of every column to see if it's safe to delete
final byte[] start = idToBytes(tree_id);
final byte[] end = idToBytes(tree_id + 1);
final Scanner scanner = tsdb.getClient().newScanner(tsdb.treeTable());
scanner.setStartKey(start);
scanner.setStopKey(end);
scanner.setFamily(TREE_FAMILY);
final Deferred<Boolean> completed = new Deferred<Boolean>();
/**
* Scanner callback that loops through all rows between tree id and
* tree id++ searching for tree related columns to delete.
*/
final class DeleteTreeScanner implements Callback<Deferred<Boolean>,
ArrayList<ArrayList<KeyValue>>> {
// list where we'll store delete requests for waiting on
private final ArrayList<Deferred<Object>> delete_deferreds =
new ArrayList<Deferred<Object>>();
/**
* Fetches the next set of rows from the scanner and adds this class as
* a callback
* @return The list of delete requests when the scanner returns a null set
*/
public Deferred<Boolean> deleteTree() {
return scanner.nextRows().addCallbackDeferring(this);
}
@Override
public Deferred<Boolean> call(ArrayList<ArrayList<KeyValue>> rows)
throws Exception {
if (rows == null) {
completed.callback(true);
return null;
}
for (final ArrayList<KeyValue> row : rows) {
// one delete request per row. We'll almost always delete the whole
// row, so just preallocate the entire row.
ArrayList<byte[]> qualifiers = new ArrayList<byte[]>(row.size());
for (KeyValue column : row) {
// tree
if (delete_definition && Bytes.equals(TREE_QUALIFIER, column.qualifier())) {
LOG.trace("Deleting tree defnition in row: " +
Branch.idToString(column.key()));
qualifiers.add(column.qualifier());
// branches
} else if (Bytes.equals(Branch.BRANCH_QUALIFIER(), column.qualifier())) {
LOG.trace("Deleting branch in row: " +
Branch.idToString(column.key()));
qualifiers.add(column.qualifier());
// leaves
} else if (column.qualifier().length > Leaf.LEAF_PREFIX().length &&
Bytes.memcmp(Leaf.LEAF_PREFIX(), column.qualifier(), 0,
Leaf.LEAF_PREFIX().length) == 0) {
LOG.trace("Deleting leaf in row: " +
Branch.idToString(column.key()));
qualifiers.add(column.qualifier());
// collisions
} else if (column.qualifier().length > COLLISION_PREFIX.length &&
Bytes.memcmp(COLLISION_PREFIX, column.qualifier(), 0,
COLLISION_PREFIX.length) == 0) {
LOG.trace("Deleting collision in row: " +
Branch.idToString(column.key()));
qualifiers.add(column.qualifier());
// not matched
} else if (column.qualifier().length > NOT_MATCHED_PREFIX.length &&
Bytes.memcmp(NOT_MATCHED_PREFIX, column.qualifier(), 0,
NOT_MATCHED_PREFIX.length) == 0) {
LOG.trace("Deleting not matched in row: " +
Branch.idToString(column.key()));
qualifiers.add(column.qualifier());
// tree rule
} else if (delete_definition && column.qualifier().length > TreeRule.RULE_PREFIX().length &&
Bytes.memcmp(TreeRule.RULE_PREFIX(), column.qualifier(), 0,
TreeRule.RULE_PREFIX().length) == 0) {
LOG.trace("Deleting tree rule in row: " +
Branch.idToString(column.key()));
qualifiers.add(column.qualifier());
}
}
if (qualifiers.size() > 0) {
final DeleteRequest delete = new DeleteRequest(tsdb.treeTable(),
row.get(0).key(), TREE_FAMILY,
qualifiers.toArray(new byte[qualifiers.size()][])
);
delete_deferreds.add(tsdb.getClient().delete(delete));
}
}
/**
* Callback used as a kind of buffer so that we don't wind up loading
* thousands or millions of delete requests into memory and possibly run
* into a StackOverflowError or general OOM. The scanner defaults are
* our limit so each pass of the scanner will wait for the previous set
* of deferreds to complete before continuing
*/
final class ContinueCB implements Callback<Deferred<Boolean>,
ArrayList<Object>> {
public Deferred<Boolean> call(ArrayList<Object> objects) {
LOG.debug("Purged [" + objects.size() + "] columns, continuing");
delete_deferreds.clear();
// call ourself again to get the next set of rows from the scanner
return deleteTree();
}
}
// call ourself again after waiting for the existing delete requests
// to complete
Deferred.group(delete_deferreds).addCallbackDeferring(new ContinueCB());
return null;
}
}
// start the scanner
new DeleteTreeScanner().deleteTree();
return completed;
}
/**
* Converts the tree ID into a byte array {@link #TREE_ID_WIDTH} in size
* @param tree_id The tree ID to convert
* @return The tree ID as a byte array
* @throws IllegalArgumentException if the Tree ID is invalid
*/
public static byte[] idToBytes(final int tree_id) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Missing or invalid tree ID");
}
final byte[] id = Bytes.fromInt(tree_id);
return Arrays.copyOfRange(id, id.length - TREE_ID_WIDTH, id.length);
}
/**
* Attempts to convert the given byte array into an integer tree ID
* <b>Note:</b> You can give this method a full branch row key and it will
* only parse out the first {@link #TREE_ID_WIDTH} bytes.
* @param row_key The row key or tree ID as a byte array
* @return The tree ID as an integer value
* @throws IllegalArgumentException if the byte array is less than
* {@link #TREE_ID_WIDTH} long
*/
public static int bytesToId(final byte[] row_key) {
if (row_key.length < TREE_ID_WIDTH) {
throw new IllegalArgumentException("Row key was less than " +
TREE_ID_WIDTH + " in length");
}
final byte[] tree_id = new byte[INT_WIDTH];
System.arraycopy(row_key, 0, tree_id, INT_WIDTH - Tree.TREE_ID_WIDTH(),
Tree.TREE_ID_WIDTH());
return Bytes.getInt(tree_id);
}
/** @return The configured collision column qualifier prefix */
public static byte[] COLLISION_PREFIX() {
return COLLISION_PREFIX;
}
/** @return The configured not-matched column qualifier prefix */
public static byte[] NOT_MATCHED_PREFIX() {
return NOT_MATCHED_PREFIX;
}
/** @return The family to use when storing tree data */
public static byte[] TREE_FAMILY() {
return TREE_FAMILY;
}
/**
* Sets or resets the changed map flags
*/
private void initializeChangedMap() {
// set changed flags
// tree_id can't change
changed.put("name", false);
changed.put("field", false);
changed.put("description", false);
changed.put("notes", false);
changed.put("strict_match", false);
changed.put("rules", false);
changed.put("not_matched", false);
changed.put("collisions", false);
changed.put("created", false);
changed.put("last_update", false);
changed.put("version", false);
changed.put("node_separator", false);
changed.put("enabled", false);
changed.put("store_failures", false);
}
/**
* Converts the object to a JSON byte array, necessary for CAS calls and to
* keep redundant data down
* @return A byte array with the serialized tree
*/
private byte[] toStorageJson() {
// TODO - precalc how much memory to grab
final ByteArrayOutputStream output = new ByteArrayOutputStream();
try {
final JsonGenerator json = JSON.getFactory().createGenerator(output);
json.writeStartObject();
// we only need to write a small amount of information
//json.writeNumberField("treeId", tree_id);
json.writeStringField("name", name);
json.writeStringField("description", description);
json.writeStringField("notes", notes);
json.writeBooleanField("strictMatch", strict_match);
json.writeNumberField("created", created);
json.writeBooleanField("enabled", enabled);
json.writeBooleanField("storeFailures", store_failures);
json.writeEndObject();
json.close();
// TODO zero copy?
return output.toByteArray();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Configures a scanner to run through all rows in the UID table that are
* {@link #TREE_ID_WIDTH} bytes wide using a row key regex filter
* @param tsdb The TSDB to use for storage access
* @return The configured HBase scanner
*/
private static Scanner setupAllTreeScanner(final TSDB tsdb) {
final byte[] start = new byte[TREE_ID_WIDTH];
final byte[] end = new byte[TREE_ID_WIDTH];
Arrays.fill(end, (byte)0xFF);
final Scanner scanner = tsdb.getClient().newScanner(tsdb.treeTable());
scanner.setStartKey(start);
scanner.setStopKey(end);
scanner.setFamily(TREE_FAMILY);
// set the filter to match only on TREE_ID_WIDTH row keys
final StringBuilder buf = new StringBuilder(20);
buf.append("(?s)" // Ensure we use the DOTALL flag.
+ "^\\Q");
buf.append("\\E(?:.{").append(TREE_ID_WIDTH).append("})$");
scanner.setKeyRegexp(buf.toString(), CHARSET);
return scanner;
}
/**
* Attempts to flush the collisions to storage. The storage call is a PUT so
* it will overwrite any existing columns, but since each column is the TSUID
* it should only exist once and the data shouldn't change.
* <b>Note:</b> This will also clear the local {@link #collisions} map
* @param tsdb The TSDB to use for storage access
* @return A meaningless deferred (will always be true since we need to group
* it with tree store calls) for the caller to wait on
* @throws HBaseException if there was an issue
*/
public Deferred<Boolean> flushCollisions(final TSDB tsdb) {
if (!store_failures) {
collisions.clear();
return Deferred.fromResult(true);
}
final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
row_key[TREE_ID_WIDTH] = COLLISION_ROW_SUFFIX;
final byte[][] qualifiers = new byte[collisions.size()][];
final byte[][] values = new byte[collisions.size()][];
int index = 0;
for (Map.Entry<String, String> entry : collisions.entrySet()) {
qualifiers[index] = new byte[COLLISION_PREFIX.length +
(entry.getKey().length() / 2)];
System.arraycopy(COLLISION_PREFIX, 0, qualifiers[index], 0,
COLLISION_PREFIX.length);
final byte[] tsuid = UniqueId.stringToUid(entry.getKey());
System.arraycopy(tsuid, 0, qualifiers[index],
COLLISION_PREFIX.length, tsuid.length);
values[index] = entry.getValue().getBytes(CHARSET);
index++;
}
final PutRequest put = new PutRequest(tsdb.treeTable(), row_key,
TREE_FAMILY, qualifiers, values);
collisions.clear();
/**
* Super simple callback used to convert the Deferred<Object> to a
* Deferred<Boolean> so that it can be grouped with other storage
* calls
*/
final class PutCB implements Callback<Deferred<Boolean>, Object> {
@Override
public Deferred<Boolean> call(Object result) throws Exception {
return Deferred.fromResult(true);
}
}
return tsdb.getClient().put(put).addCallbackDeferring(new PutCB());
}
/**
* Attempts to flush the non-matches to storage. The storage call is a PUT so
* it will overwrite any existing columns, but since each column is the TSUID
* it should only exist once and the data shouldn't change.
* <b>Note:</b> This will also clear the local {@link #not_matched} map
* @param tsdb The TSDB to use for storage access
* @return A meaningless deferred (will always be true since we need to group
* it with tree store calls) for the caller to wait on
* @throws HBaseException if there was an issue
*/
public Deferred<Boolean> flushNotMatched(final TSDB tsdb) {
if (!store_failures) {
not_matched.clear();
return Deferred.fromResult(true);
}
final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
row_key[TREE_ID_WIDTH] = NOT_MATCHED_ROW_SUFFIX;
final byte[][] qualifiers = new byte[not_matched.size()][];
final byte[][] values = new byte[not_matched.size()][];
int index = 0;
for (Map.Entry<String, String> entry : not_matched.entrySet()) {
qualifiers[index] = new byte[NOT_MATCHED_PREFIX.length +
(entry.getKey().length() / 2)];
System.arraycopy(NOT_MATCHED_PREFIX, 0, qualifiers[index], 0,
NOT_MATCHED_PREFIX.length);
final byte[] tsuid = UniqueId.stringToUid(entry.getKey());
System.arraycopy(tsuid, 0, qualifiers[index],
NOT_MATCHED_PREFIX.length, tsuid.length);
values[index] = entry.getValue().getBytes(CHARSET);
index++;
}
final PutRequest put = new PutRequest(tsdb.treeTable(), row_key,
TREE_FAMILY, qualifiers, values);
not_matched.clear();
/**
* Super simple callback used to convert the Deferred<Object> to a
* Deferred<Boolean> so that it can be grouped with other storage
* calls
*/
final class PutCB implements Callback<Deferred<Boolean>, Object> {
@Override
public Deferred<Boolean> call(Object result) throws Exception {
return Deferred.fromResult(true);
}
}
return tsdb.getClient().put(put).addCallbackDeferring(new PutCB());
}
// GETTERS AND SETTERS ----------------------------
/** @return The width of the tree ID in bytes */
public static int TREE_ID_WIDTH() {
return TREE_ID_WIDTH;
}
/** @return The treeId */
public int getTreeId() {
return tree_id;
}
/** @return The name of the tree */
public String getName() {
return name;
}
/** @return An optional description of the tree */
public String getDescription() {
return description;
}
/** @return Optional notes about the tree */
public String getNotes() {
return notes;
}
/** @return Whether or not strict matching is enabled */
public boolean getStrictMatch() {
return strict_match;
}
/** @return Whether or not the tree should process TSMeta objects */
public boolean getEnabled() {
return enabled;
}
/** @return Whether or not to store not matched and collisions */
public boolean getStoreFailures() {
return store_failures;
}
/** @return The tree's rule set */
public Map<Integer, TreeMap<Integer, TreeRule>> getRules() {
return rules;
}
/** @return List of TSUIDs that did not match any rules */
@JsonIgnore
public Map<String, String> getNotMatched() {
return not_matched;
}
/** @return List of TSUIDs that were not stored due to collisions */
@JsonIgnore
public Map<String, String> getCollisions() {
return collisions;
}
/** @return When the tree was created, Unix epoch in seconds */
public long getCreated() {
return created;
}
/** @param name A descriptive name for the tree */
public void setName(String name) {
if (!this.name.equals(name)) {
changed.put("name", true);
this.name = name;
}
}
/** @param description A brief description of the tree */
public void setDescription(String description) {
if (!this.description.equals(description)) {
changed.put("description", true);
this.description = description;
}
}
/** @param notes Optional notes about the tree */
public void setNotes(String notes) {
if (!this.notes.equals(notes)) {
changed.put("notes", true);
this.notes = notes;
}
}
/** @param strict_match Whether or not a TSUID must match all rules in the
* tree to be included */
public void setStrictMatch(boolean strict_match) {
changed.put("strict_match", true);
this.strict_match = strict_match;
}
/** @param enabled Whether or not this tree should process TSMeta objects */
public void setEnabled(boolean enabled) {
this.enabled = enabled;
changed.put("enabled", true);
}
/** @param store_failures Whether or not to store not matched or collisions */
public void setStoreFailures(boolean store_failures) {
this.store_failures = store_failures;
changed.put("store_failures", true);
}
/** @param treeId ID of the tree, users cannot modify this */
public void setTreeId(int treeId) {
this.tree_id = treeId;
}
/** @param created The time when this tree was created,
* Unix epoch in seconds */
public void setCreated(long created) {
this.created = created;
}
}