/* ========================
* JSynoptic : a free Synoptic editor
* ========================
*
* Project Info: http://jsynoptic.sourceforge.net/index.html
*
* This program is free software; you can redistribute it and/or modify it under the terms
* of the GNU Lesser General Public License as published by the Free Software Foundation;
* either version 2.1 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License along with this
* program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
* Boston, MA 02111-1307, USA.
*
* (C) Copyright 2001-2006, by :
* Corporate:
* EADS Astrium SAS
* EADS CRC
* Individual:
* Claude Cazenave
*
* $Id: AsynchronousMergeDSCollection.java,v 1.7 2008/04/08 12:00:36 ogor Exp $
*
* Changes
* -------
* 20 janv. 2006 : Initial public release (CC);
*
*/
package simtools.data.async;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.DoubleBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.logging.Logger;
import simtools.data.CollectiveDataSource;
import simtools.data.DataException;
import simtools.data.DataInfo;
import simtools.data.DataSource;
import simtools.data.DataSourceCollection;
import simtools.data.DataSourcePool;
import simtools.data.UnsupportedOperation;
import simtools.data.async.TimeStampedDataSource.TimeSource;
import simtools.data.merge.MergeDSCollection;
import simtools.data.merge.MergeDataException;
/**
* Merge data in a asynchronous way. each member of
* AsynchronousMergeDSCollection is stamped with a time data.
*
* @author zxpletran007
*/
public class AsynchronousMergeDSCollection extends TimeStampedDataSourceCollection implements MergeDSCollection {
/**
* A looger to dump error or warning messages in a soket, an output stream,
* a file...
*/
protected static Logger _logger = simtools.util.LogConfigurator.getLogger(AsynchronousMergeDSCollection.class
.getName());
public static final String ID_MARKER = "AsynchronousMergeDSCollection:";
// If a TM repository has more than hugeRepertorySize data sources, it can
// be structured in an sub arborescence.
// Thus sub repositories are created with the different first letters of
// thoses data sources.
protected final static int HUGEREPERTORYSIZE = 10;
protected MergedCollectionContainer collectionRoot;
protected String collectionName;
public AsynchronousMergeDSCollection(String collectionName) throws IOException, InvalidFormatException {
super();
this.collectionName = collectionName;
collectionRoot = new MergedCollectionContainer("");
map = new HashMap();
}
public Object getValue(int i, long index) throws DataException {
return ((DataSource) get(i)).getValue(index);
}
public double getDoubleValue(int i, long index) throws DataException {
return ((DataSource) get(i)).getDoubleValue(index);
}
public DataInfo getInformation() {
return new DataInfo(collectionName, ID_MARKER + collectionName, "Asynchronous merge");
}
public String getCollectionName() {
return collectionName;
}
/*
* (non-Javadoc)
*
* @see simtools.data.merge.MergeDSCollection#add(simtools.data.async.TimeStampedDataSourceCollection)
*/
public void add(TimeStampedDataSourceCollection tsdsc, double offset, double initialDate) throws MergeDataException {
if ((tsdsc instanceof MergeDSCollection)) {
throw new MergeDataException(DataInfo.getId(tsdsc) + " is already a merge of collection");
}
AsynchronousMergeDataSource mds = null;
for (int i = 0; i < tsdsc.size(); i++) {
try{
if (!(tsdsc.get(i) instanceof TimeStampedDataSource)) {
continue;
}
TimeStampedDataSource tsds = (TimeStampedDataSource) tsdsc.get(i);
String id = DataInfo.getId(tsds);
if (get(id) != null) {
id += "_" + DataInfo.getLabel(tsdsc); // If data source name already exists then add collection name as a suffix
}
if (get(id) == null) {
MergedCollectionContainer node = getDataSourceNode(id);
String label = id.substring(id.lastIndexOf(".") + 1);
mds = new AsynchronousMergeDataSource(label, id, tsds, tsds.getTime(), false, offset, 0);
node.addDataSource(mds);
add(mds);
map.put(id, mds);
}
} catch (DataException e){
MergeDataException.mergeDataErrors.add(e.getMessage());
} catch (IOException e){
MergeDataException.mergeDataErrors.add(e.getMessage());
} catch (InvalidFormatException e){
MergeDataException.mergeDataErrors.add(e.getMessage());
}
}
// Update pool with last data
DataSourcePool.global.DataSourceCollectionDataSourceAdded(this, mds);
sortHugeRepertories();
}
/*
* (non-Javadoc)
*
* @see simtools.data.merge.MergeDSCollection#add(simtools.data.async.TimeStampedDataSource)
*/
public void add(TimeStampedDataSource ds, double offset, double initialDate) throws MergeDataException {
add(ds, ds.getTime(), false, offset, initialDate);
}
/*
* (non-Javadoc)
*
* @see simtools.data.merge.MergeDSCollection#add(simtools.data.DataSourceCollection,
* simtools.data.DataSource, double)
*/
public void add(DataSourceCollection dsc, DataSource timeRef, boolean isRelative, double offset, double initialDate)
throws MergeDataException {
if (dsc instanceof MergeDSCollection) {
throw new MergeDataException(DataInfo.getId(dsc) + " is already a merge of collection");
}
// Same time reference for all created synchronous time stamped data
TimeSource timeReference = null;
AsynchronousMergeDataSource mds = null;
for (int i = 0; i < dsc.size(); i++) {
try {
DataSource data = (DataSource) dsc.get(i);
String id = DataInfo.getId(data);
if (!(id.equals(DataInfo.getId(timeRef)))) { // Do not add the time reference to merge collection
if (get(id) != null) {
id += "_" + DataInfo.getLabel(dsc); // if data source name already exists then add collection name as a suffix
}
if (get(id) == null) {
MergedCollectionContainer node = getDataSourceNode(id);
String label = id.substring(id.lastIndexOf(".") + 1);
if (timeReference == null) {
mds = new AsynchronousMergeDataSource(label, id, data, timeRef, isRelative, offset,
initialDate);
timeReference = mds.ref;
} else {
mds = new AsynchronousMergeDataSource(label, id, data, timeReference);
}
node.addDataSource(mds);
int s = size();
add(mds);
map.put(DataInfo.getId(mds), mds);
}
}
} catch (InvalidFormatException e) {
MergeDataException.mergeDataErrors.add(e.getMessage());
} catch (IOException e) {
MergeDataException.mergeDataErrors.add(e.getMessage());
} catch (DataException e) {
MergeDataException.mergeDataErrors.add(e.getMessage());
}
}
// Update pool with last data
if (mds != null) {
DataSourcePool.global.DataSourceCollectionDataSourceAdded(this, mds);
sortHugeRepertories();
}
}
public void add(DataSource ds, DataSource timeRef, boolean isRelative, double offset, double initialDate)
throws MergeDataException {
try {
String id = DataInfo.getId(ds);
if (!(id.equals(DataInfo.getId(timeRef)))) { // Do not add the time
// reference to merge
// collection
if (get(id) != null) { // if data source name already exists then
// add collection name as a suffix
String name;
if (ds instanceof CollectiveDataSource) {
name = DataInfo.getLabel(((CollectiveDataSource) ds).getCollection());
} else {
name = DataInfo.getLabel(DataSourcePool.global.getCollectionForDataSourceId(DataInfo.getId(ds)));
}
id += "_" + name;
}
// If new name dosn't exist into collection, add data source
if (get(id) == null) {
MergedCollectionContainer node = getDataSourceNode(id);
String label = id.substring(id.lastIndexOf(".") + 1);
AsynchronousMergeDataSource mds;
try {
mds = new AsynchronousMergeDataSource(label, id, ds, timeRef,
isRelative, offset, initialDate);
} catch (DataException e) {
throw new MergeDataException(e.getMessage());
}
node.addDataSource(mds);
add(mds);
map.put(DataInfo.getId(mds), mds);
DataSourcePool.global.DataSourceCollectionDataSourceAdded(this, mds);
sortHugeRepertories();
}
}
}catch (Exception e){
throw new MergeDataException(e.getMessage());
}
}
public boolean isCompound() {
return true;
}
public Collection getCollectionContainers() {
return collectionRoot.getChildren();
}
public class MergedCollectionContainer extends Container {
/**
* Create a new container with the given value
*/
public MergedCollectionContainer(String value) {
super(value);
children = new ArrayList();
}
public Collection getChildren() {
return children;
}
public void setChildren(Collection c) {
children = c;
}
protected MergedCollectionContainer getChildCollection(String childName) {
Object o;
for (Iterator iter = children.iterator(); iter.hasNext();) {
o = iter.next();
if (o instanceof MergedCollectionContainer) {
MergedCollectionContainer c = (MergedCollectionContainer) o;
if (c.value.equals(childName)) {
return c;
}
}
}
return null;
}
protected void removeCollectionArborescence() {
Object o;
for (Iterator iter = children.iterator(); iter.hasNext();) {
o = iter.next();
if (o instanceof MergedCollectionContainer) {
((MergedCollectionContainer) o).removeCollectionArborescence();
}
}
children.clear();
}
}
/**
* Get the container where data source can be added
*
* @param name.
* data source name
* @return <b>(OCContainer)</b>.The container where data source can be
* added
*/
protected MergedCollectionContainer getDataSourceNode(String name) {
MergedCollectionContainer currentDirectory = collectionRoot;
String prefixe, sufixe;
int lengthNameModel = name.indexOf(".");
// Placement de la DS dans une collection fille
while (lengthNameModel != -1) {
prefixe = name.substring(0, lengthNameModel);
sufixe = name.substring(lengthNameModel + 1, name.length());
MergedCollectionContainer subCollection = currentDirectory.getChildCollection(prefixe);
if (subCollection == null) {
MergedCollectionContainer lastDirectory = new MergedCollectionContainer(prefixe);
currentDirectory.addContainer(lastDirectory);
currentDirectory = lastDirectory;
} else {
currentDirectory = subCollection;
}
name = sufixe;
lengthNameModel = name.indexOf(".");
}
return currentDirectory;
}
protected void sortHugeRepertories() {
// We compute all 2nd levels -> ex : PL.*
if (((collectionRoot.getChildren().iterator().hasNext()) && ((collectionRoot.getChildren().iterator().next()) instanceof MergedCollectionContainer))) {
Collection subRoots = ((MergedCollectionContainer) (collectionRoot.getChildren().iterator().next()))
.getChildren();
Object o;
for (Iterator iter = subRoots.iterator(); iter.hasNext();) {
o = iter.next();
if (o instanceof MergedCollectionContainer) {
sortHugeRepertory((MergedCollectionContainer) o, 2);
}
}
}
}
/**
* @param repertory ,
* where data source have to be structured
* @param proof,
* number of prefixes to remove from all data sources names, in
* order to compute the new structure. Ex : if PL.TM.variable ->
* proof = 2
*/
protected void sortHugeRepertory(MergedCollectionContainer repertory, int proof) {
Object o;
int count = 0;
String start = null;
int diff = 0;
for (Iterator iter = repertory.getChildren().iterator(); iter.hasNext();) {
o = iter.next();
if (o instanceof DataSource) {
String dsName = ((DataSource) o).getInformation().id;
String repName = dsName.substring(dsName.indexOf(".") + 1);
for (int i = 0; i < proof - 1; i++) {
repName = repName.substring(repName.indexOf(".") + 1);
}
if (start == null) {
start = repName.substring(0, repName.length() > 2 ? 3 : repName.length());
} else {
if (!repName.startsWith(start)) {
diff++;
}
}
count++;
}
}
if ((count > HUGEREPERTORYSIZE) && (diff > 0)) {
Hashtable newRepositories = new Hashtable();
ArrayList dataList = new ArrayList(repertory.getChildren());
for (Iterator iter = repertory.getChildren().iterator(); iter.hasNext();) {
o = iter.next();
if (o instanceof DataSource) {
DataSource d = (DataSource) o;
String dsName = ((d.getInformation().id));
String repName = dsName.substring(dsName.indexOf(".") + 1);
for (int i = 0; i < proof - 1; i++) {
repName = repName.substring(repName.indexOf(".") + 1);
}
repName = repName.substring(0, 1);
MergedCollectionContainer rep;
if (newRepositories.containsKey(repName)) {
rep = (MergedCollectionContainer) newRepositories.get(repName);
} else {
rep = new MergedCollectionContainer(repName);
newRepositories.put(repName, rep);
dataList.add(rep);
}
rep.addDataSource(d);
dataList.remove(d);
}
}
repertory.setChildren(dataList);
}
}
public class AsynchronousMergeDataSource extends TimeStampedDataSource {
protected final DataSource source;
protected final TimeSource ref;
public AsynchronousMergeDataSource(String label, String id, DataSource source, DataSource timeSource,
boolean isRelative, double offset, double initialDate) throws DataException, IOException,
InvalidFormatException {
super(label, id, AsynchronousMergeDSCollection.this);
ref = new RefTime(timeSource, isRelative, offset, initialDate);
setTime(ref);
this.source = source;
setInfo(id);
}
public AsynchronousMergeDataSource(String label, String id, DataSource source, TimeSource ref)
throws IOException, InvalidFormatException {
super(label, id, AsynchronousMergeDSCollection.this);
setTime(ref);
this.ref = ref;
this.source = source;
setInfo(id);
}
protected void setInfo(String name) {
getInformation().unit = DataInfo.getUnit(source);
getInformation().comment = "Asynchronous merge from data source:" + DataInfo.getId(source);
}
/*
* (non-Javadoc)
*
* @see simtools.data.async.TimeStampedDataSource#getStart()
*/
public double getStart() {
try {
return ref.getDoubleValue(ref.getStartIndex());
} catch (DataException e) {
e.printStackTrace();
return 0.;
}
}
/*
* (non-Javadoc)
*
* @see simtools.data.async.TimeStampedDataSource#getEnd()
*/
public double getEnd() {
try {
return ref.getDoubleValue(ref.getLastIndex());
} catch (DataException e) {
e.printStackTrace();
return 0.;
}
}
protected void computeMinMax() throws UnsupportedOperation {
source.computeMin();
source.computeMax();
}
public Object getValue(long index) throws DataException {
return source.getValue(index);
}
public double getDoubleValue(long index) throws DataException {
return source.getDoubleValue(index);
}
/*
* (non-Javadoc)
*
* @see simtools.data.async.TimeStampedDataSource#getStartIndex()
*/
public long getStartIndex() throws UnsupportedOperation {
return source.getStartIndex();
}
/*
* (non-Javadoc)
*
* @see simtools.data.DataSource#getLastIndex()
*/
public long getLastIndex() throws UnsupportedOperation {
return source.getLastIndex();
}
/*
* (non-Javadoc)
*
* @see simtools.data.async.TimeStampedDataSource#isValid()
*/
public boolean isValid() {
return true;
}
/*
* (non-Javadoc)
*
* @see simtools.data.DataSource#getDoubleMin()
*/
public double getDoubleMin() throws DataException {
return source.getDoubleMin();
}
/*
* (non-Javadoc)
*
* @see simtools.data.DataSource#getDoubleMin()
*/
public double getDoubleMax() throws DataException {
return source.getDoubleMax();
}
/*
* (non-Javadoc)
*
* @see simtools.data.async.TimeStampedDataSource#getMax()
*/
public Object getMax() {
try {
return source.getMax();
} catch (UnsupportedOperation e) {
return null;
}
}
/*
* (non-Javadoc)
*
* @see simtools.data.async.TimeStampedDataSource#getMin()
*/
public Object getMin() {
try {
return source.getMin();
} catch (UnsupportedOperation e) {
return null;
}
}
public class RefTime extends TimeSource {
final DoubleBuffer db;
final long startIndex;
final long lastIndex;
/**
* @param timeSource
* @param offset.
* A number of seconds
* @throws DataException
*/
public RefTime(DataSource timeSource, boolean isRelative, double offset, double initialDate)
throws DataException {
startIndex = timeSource.getStartIndex();
lastIndex = timeSource.getLastIndex();
long length = lastIndex - startIndex + 1;
if ((length * 8) > Integer.MAX_VALUE) {
throw new IllegalArgumentException(DataInfo.getId(timeSource) + "has too many data:" + length);
}
if (length > 10000) { // native allocation
final ByteBuffer bb = ByteBuffer.allocateDirect(8 * (int) length);
db = bb.asDoubleBuffer();
} else { // heap allocation
db = DoubleBuffer.allocate((int) length);
}
// Offset conversion : seconds --> milli seconds
offset *= 1000;
for (int i = 0; i < length; i++) {
double timeValue;
if (isRelative) {
timeValue = timeSource.getDoubleValue(startIndex + i) * 1000 + initialDate + offset;
} else {
timeValue = timeSource.getDoubleValue(startIndex + i) + offset;
}
db.put(timeValue);
}
getInformation().comment = "Reference used for time is " + DataInfo.getId(timeSource) + "\nOffset is "
+ offset;
}
/*
* (non-Javadoc)
*
* @see simtools.data.DataSource#getStartIndex()
*/
public long getStartIndex() throws UnsupportedOperation {
return startIndex;
}
/*
* (non-Javadoc)
*
* @see simtools.data.DataSource#getLastIndex()
*/
public long getLastIndex() throws UnsupportedOperation {
return lastIndex;
}
public Object getValue(long index) throws DataException {
long k = index - startIndex;
if (k > Integer.MAX_VALUE) {
throw new DataException("invalid index : " + index + " startIndex=" + startIndex);
}
return new Double(db.get((int) k));
}
public double getDoubleValue(long index) throws DataException {
long k = index - startIndex;
if (k > Integer.MAX_VALUE) {
throw new DataException("invalid index : " + index + " startIndex=" + startIndex);
}
return (db.get((int) k));
}
}
}
}